Merge branch 'master' of github.com:zalando/patroni into feature/quorum-commit

PatchStream class was removed as it was effectively doing nothing useful.

.github/workflows/install_deps.py

@@ -46,7 +46,7 @@ def install_packages(what):
     packages = packages.get(what, [])
     ver = versions.get(what)
     if float(ver) >= 15:
-        packages += ['postgresql-{0}-citus-11.2'.format(ver)]
+        packages += ['postgresql-{0}-citus-12.1'.format(ver)]
     subprocess.call(['sudo', 'apt-get', 'update', '-y'])
     return subprocess.call(['sudo', 'apt-get', 'install', '-y', 'postgresql-' + ver, 'expect-dev'] + packages)
 
@@ -110,7 +110,7 @@ def install_etcd():
 
 
 def install_postgres():
-    version = os.environ.get('PGVERSION', '15.1-1')
+    version = os.environ.get('PGVERSION', '16.1-1')
     platform = {'darwin': 'osx', 'win32': 'windows-x64', 'cygwin': 'windows-x64'}[sys.platform]
     if platform == 'osx':
         return subprocess.call(['brew', 'install', 'expect', 'postgresql@{0}'.format(version.split('.')[0])])

.github/workflows/mapping.py

@@ -1 +1 @@
-versions = {'etcd': '9.6', 'etcd3': '14', 'consul': '13', 'exhibitor': '12', 'raft': '11', 'kubernetes': '15'}
+versions = {'etcd': '9.6', 'etcd3': '16', 'consul': '13', 'exhibitor': '12', 'raft': '14', 'kubernetes': '15'}

.github/workflows/release.yaml

@@ -24,8 +24,11 @@ jobs:
     - name: Run tests and flake8
       run: python .github/workflows/run_tests.py
 
+    - name: Install Python packaging build frontend
+      run: python -m pip install build
+
     - name: Build a binary wheel and a source tarball
-      run: python setup.py sdist bdist_wheel
+      run: python -m build
 
     - name: Publish distribution to Test PyPI
       if: github.event_name == 'push'

.github/workflows/run_tests.py

@@ -30,7 +30,7 @@ def main():
         unbuffer = ['timeout', '900', 'unbuffer']
     else:
         if sys.platform == 'darwin':
-            version = os.environ.get('PGVERSION', '15.1-1')
+            version = os.environ.get('PGVERSION', '16.1-1')
             path = '/usr/local/opt/postgresql@{0}/bin:.'.format(version.split('.')[0])
             unbuffer = ['unbuffer']
         else:

.github/workflows/tests.yaml

@@ -5,6 +5,7 @@ on:
   push:
     branches:
     - master
+    - 'REL_[0-9]+_[0-9]+'
 
 env:
   CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}
@@ -84,7 +85,7 @@ jobs:
     env:
       DCS: ${{ matrix.dcs }}
       ETCDVERSION: 3.4.23
-      PGVERSION: 15.1-1  # for windows and macos
+      PGVERSION: 16.1-1  # for windows and macos
     strategy:
       fail-fast: false
       matrix:
@@ -173,4 +174,28 @@ jobs:
 
     - uses: jakebailey/pyright-action@v1
       with:
-        version: 1.1.317
+        version: 1.1.347
+
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.11
+        cache: pip
+
+    - name: Install dependencies
+      run: pip install tox
+
+    - name: Install package dependencies
+      run: |
+        sudo apt update \
+        && sudo apt install -y \
+            latexmk texlive-latex-extra tex-gyre \
+            --no-install-recommends
+
+    - name: Generate documentation
+      run: tox -m docs

.gitignore

@@ -27,14 +27,15 @@ lib64
 pip-log.txt
 
 # Unit test / coverage reports
-.coverage
+.coverage*
 .tox
 nosetests.xml
 coverage.xml
 htmlcov
 junit.xml
-features/output
+features/output*
 dummy
+result.json
 
 # Translations
 *.mo
@@ -48,9 +49,12 @@ pgpass
 scm-source.json
 
 # Sphinx-generated documentation
+docs/_build/
 docs/build/
 docs/source/_static/
 docs/source/_templates/
+docs/modules/
+docs/pdf/
 
 # Pycharm IDE
 .idea/
@@ -63,3 +67,6 @@ venv*/
 
 # Default test data directory
 data/
+
+# macOS
+**/.DS_Store

.readthedocs.yaml

@@ -19,3 +19,8 @@ formats:
   - epub
   - pdf
   - htmlzip
+
+python:
+  install:
+    - requirements: requirements.docs.txt
+    - requirements: requirements.txt

Dockerfile

@@ -1,6 +1,6 @@
 ## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
 ## It has all the necessary components to play/debug with a single node appliance, running etcd
-ARG PG_MAJOR=15
+ARG PG_MAJOR=16
 ARG COMPRESS=false
 ARG PGHOME=/home/postgres
 ARG PGDATA=$PGHOME/data
@@ -25,8 +25,7 @@ RUN set -ex \
             | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \
             | xargs apt-get install -y vim curl less jq locales haproxy sudo \
                             python3-etcd python3-kazoo python3-pip busybox \
-                            net-tools iputils-ping --fix-missing \
+                            net-tools iputils-ping dumb-init --fix-missing \
-    && pip3 install dumb-init \
 \
     # Cleanup all locales but en_US.UTF-8
     && find /usr/share/i18n/charmaps/ -type f ! -name UTF-8.gz -delete \
@@ -71,7 +70,7 @@ RUN set -ex \
     # Clean up all useless packages and some files
     && apt-get purge -y --allow-remove-essential python3-pip gzip bzip2 util-linux e2fsprogs \
                 libmagic1 bsdmainutils login ncurses-bin libmagic-mgc e2fslibs bsdutils \
-                exim4-config gnupg-agent dirmngr libpython2.7-stdlib libpython2.7-minimal \
+                exim4-config gnupg-agent dirmngr \
                 git make \
     && apt-get autoremove -y \
     && apt-get clean -y \
@@ -95,9 +94,9 @@ RUN set -ex \
         /usr/share/locale/??_?? \
         /usr/share/postgresql/*/man \
         /usr/share/postgresql-common/pg_wrapper \
-        /usr/share/vim/vim80/doc \
+        /usr/share/vim/vim*/doc \
-        /usr/share/vim/vim80/lang \
+        /usr/share/vim/vim*/lang \
-        /usr/share/vim/vim80/tutor \
+        /usr/share/vim/vim*/tutor \
 #        /var/lib/dpkg/info/* \
     && find /usr/bin -xtype l -delete \
     && find /var/log -type f -exec truncate --size 0 {} \; \
@@ -126,6 +125,8 @@ RUN if [ "$COMPRESS" = "true" ]; then \
         && /bin/busybox sh -c "(find $save_dirs -not -type d && cat /exclude /exclude && echo exclude) | sort | uniq -u | xargs /bin/busybox rm" \
         && /bin/busybox --install -s \
         && /bin/busybox sh -c "find $save_dirs -type d -depth -exec rmdir -p {} \; 2> /dev/null"; \
+    else \
+        /bin/busybox --install -s; \
     fi
 
 FROM scratch
@@ -144,6 +145,7 @@ ARG PGBIN=/usr/lib/postgresql/$PG_MAJOR/bin
 
 ENV LC_ALL=$LC_ALL LANG=$LANG EDITOR=/usr/bin/editor
 ENV PGDATA=$PGDATA PATH=$PATH:$PGBIN
+ENV ETCDCTL_API=3
 
 COPY patroni /patroni/
 COPY extras/confd/conf.d/haproxy.toml /etc/confd/conf.d/

Dockerfile.citus

@@ -1,6 +1,6 @@
 ## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
 ## It has all the necessary components to play/debug with a single node appliance, running etcd
-ARG PG_MAJOR=15
+ARG PG_MAJOR=16
 ARG COMPRESS=false
 ARG PGHOME=/home/postgres
 ARG PGDATA=$PGHOME/data
@@ -25,7 +25,7 @@ RUN set -ex \
             | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \
             | xargs apt-get install -y vim curl less jq locales haproxy sudo \
                             python3-etcd python3-kazoo python3-pip busybox \
-                            net-tools iputils-ping lsb-release --fix-missing \
+                            net-tools iputils-ping lsb-release dumb-init --fix-missing \
     && if [ $(dpkg --print-architecture) = 'arm64' ]; then \
         apt-get install -y postgresql-server-dev-$PG_MAJOR \
                            git gcc make autoconf \
@@ -40,9 +40,8 @@ RUN set -ex \
         echo "deb [signed-by=/etc/apt/trusted.gpg.d/citusdata_community.gpg] https://packagecloud.io/citusdata/community/debian/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/citusdata_community.list \
         && curl -sL https://packagecloud.io/citusdata/community/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/citusdata_community.gpg \
         && apt-get update -y \
-        && apt-get -y install postgresql-$PG_MAJOR-citus-11.3; \
+        && apt-get -y install postgresql-$PG_MAJOR-citus-12.1; \
     fi \
-    && pip3 install dumb-init \
 \
     # Cleanup all locales but en_US.UTF-8
     && find /usr/share/i18n/charmaps/ -type f ! -name UTF-8.gz -delete \
@@ -88,7 +87,7 @@ RUN set -ex \
     # Clean up all useless packages and some files
     && apt-get purge -y --allow-remove-essential python3-pip gzip bzip2 util-linux e2fsprogs \
                 libmagic1 bsdmainutils login ncurses-bin libmagic-mgc e2fslibs bsdutils \
-                exim4-config gnupg-agent dirmngr libpython2.7-stdlib libpython2.7-minimal \
+                exim4-config gnupg-agent dirmngr \
                 postgresql-server-dev-$PG_MAJOR git gcc make autoconf \
                 libc6-dev flex libicu-dev libkrb5-dev liblz4-dev \
                 libpam0g-dev libreadline-dev libselinux1-dev libssl-dev libxslt1-dev libzstd-dev uuid-dev \
@@ -114,9 +113,9 @@ RUN set -ex \
         /usr/share/locale/??_?? \
         /usr/share/postgresql/*/man \
         /usr/share/postgresql-common/pg_wrapper \
-        /usr/share/vim/vim80/doc \
+        /usr/share/vim/vim*/doc \
-        /usr/share/vim/vim80/lang \
+        /usr/share/vim/vim*/lang \
-        /usr/share/vim/vim80/tutor \
+        /usr/share/vim/vim*/tutor \
 #        /var/lib/dpkg/info/* \
     && find /usr/bin -xtype l -delete \
     && find /var/log -type f -exec truncate --size 0 {} \; \
@@ -165,6 +164,7 @@ ARG PGBIN=/usr/lib/postgresql/$PG_MAJOR/bin
 
 ENV LC_ALL=$LC_ALL LANG=$LANG EDITOR=/usr/bin/editor
 ENV PGDATA=$PGDATA PATH=$PATH:$PGBIN
+ENV ETCDCTL_API=3
 
 COPY patroni /patroni/
 COPY extras/confd/conf.d/haproxy.toml /etc/confd/conf.d/

README.rst

@@ -12,7 +12,7 @@ Patroni is a template for high availability (HA) PostgreSQL solutions using Pyth
 
 We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely.
 
-Currently supported PostgreSQL versions: 9.3 to 15.
+Currently supported PostgreSQL versions: 9.3 to 16.
 
 **Note to Citus users**: Starting from 3.0 Patroni nicely integrates with the `Citus <https://github.com/citusdata/citus>`__ database extension to Postgres. Please check the `Citus support page <https://github.com/zalando/patroni/blob/master/docs/citus.rst>`__ in the Patroni documentation for more info about how to use Patroni high availability together with a Citus distributed cluster.
 
@@ -74,27 +74,11 @@ There are a few options available:
 
 ::
 
-    sudo apt-get install python-psycopg2   # install python2 psycopg2 module on Debian/Ubuntu
+    sudo apt-get install python3-psycopg2  # install psycopg2 module on Debian/Ubuntu
-    sudo apt-get install python3-psycopg2  # install python3 psycopg2 module on Debian/Ubuntu
+    sudo yum install python3-psycopg2      # install psycopg2 on RedHat/Fedora/CentOS
-    sudo yum install python-psycopg2       # install python2 psycopg2 on RedHat/Fedora/CentOS
 
-2. Install psycopg2 from the binary package
+2. Specify one of `psycopg`, `psycopg2`, or `psycopg2-binary` in the list of dependencies when installing Patroni with pip (see below).
 
-::
-
-    pip install psycopg2-binary
-
-3. Install psycopg2 from source
-
-::
-
-    pip install psycopg2>=2.5.4
-
-4. Use psycopg 3.0 instead of psycopg2
-
-::
-
-    pip install psycopg[binary]
 
 **General installation for pip**
 
@@ -120,12 +104,20 @@ raft
     `pysyncobj` module in order to use python Raft implementation as DCS
 aws
     `boto3` in order to use AWS callbacks
+all
+    all of the above (except psycopg family)
+psycopg3
+    `psycopg[binary]>=3.0.0` module
+psycopg2
+    `psycopg2>=2.5.4` module
+psycopg2-binary
+    `psycopg2-binary` module
 
-For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is:
+For example, the command in order to install Patroni together with psycopg3, dependencies for Etcd as a DCS, and AWS callbacks is:
 
 ::
 
-    pip install patroni[etcd,aws]
+    pip install patroni[psycopg3,etcd3,aws]
 
 Note that external tools to call in the replica creation or custom bootstrap scripts (i.e. WAL-E) should be installed independently of Patroni.
 
@@ -159,7 +151,7 @@ run:
 YAML Configuration
 ==================
 
-Go `here <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
+Go `here <https://github.com/zalando/patroni/blob/master/docs/dynamic_configuration.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
 
 =========================
 Environment Configuration

docker-compose-citus.yml

@@ -19,7 +19,6 @@ services:
         image: ${PATRONI_TEST_IMAGE:-patroni-citus}
         networks: [ demo ]
         environment:
-            ETCDCTL_API: 3
             ETCD_LISTEN_PEER_URLS: http://0.0.0.0:2380
             ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379
             ETCD_INITIAL_CLUSTER: etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380
@@ -28,19 +27,19 @@ services:
             ETCD_UNSUPPORTED_ARCH: arm64
         container_name: demo-etcd1
         hostname: etcd1
-        command: etcd -name etcd1 -initial-advertise-peer-urls http://etcd1:2380
+        command: etcd --name etcd1 --initial-advertise-peer-urls http://etcd1:2380
 
     etcd2:
         <<: *etcd
         container_name: demo-etcd2
         hostname: etcd2
-        command: etcd -name etcd2 -initial-advertise-peer-urls http://etcd2:2380
+        command: etcd --name etcd2 --initial-advertise-peer-urls http://etcd2:2380
 
     etcd3:
         <<: *etcd
         container_name: demo-etcd3
         hostname: etcd3
-        command: etcd -name etcd3 -initial-advertise-peer-urls http://etcd3:2380
+        command: etcd --name etcd3 --initial-advertise-peer-urls http://etcd3:2380
 
     haproxy:
         image: ${PATRONI_TEST_IMAGE:-patroni-citus}
@@ -53,7 +52,6 @@ services:
             - "5001:5001"  # Load-balancing across workers primaries
         command: haproxy
         environment: &haproxy_env
-            ETCDCTL_API: 3
             ETCDCTL_ENDPOINTS: http://etcd1:2379,http://etcd2:2379,http://etcd3:2379
             PATRONI_ETCD3_HOSTS: "'etcd1:2379','etcd2:2379','etcd3:2379'"
             PATRONI_SCOPE: demo

docker-compose.yml

@@ -25,19 +25,19 @@ services:
             ETCD_UNSUPPORTED_ARCH: arm64
         container_name: demo-etcd1
         hostname: etcd1
-        command: etcd -name etcd1 -initial-advertise-peer-urls http://etcd1:2380
+        command: etcd --name etcd1 --initial-advertise-peer-urls http://etcd1:2380
 
     etcd2:
         <<: *etcd
         container_name: demo-etcd2
         hostname: etcd2
-        command: etcd -name etcd2 -initial-advertise-peer-urls http://etcd2:2380
+        command: etcd --name etcd2 --initial-advertise-peer-urls http://etcd2:2380
 
     etcd3:
         <<: *etcd
         container_name: demo-etcd3
         hostname: etcd3
-        command: etcd -name etcd3 -initial-advertise-peer-urls http://etcd3:2380
+        command: etcd --name etcd3 --initial-advertise-peer-urls http://etcd3:2380
 
     haproxy:
         image: ${PATRONI_TEST_IMAGE:-patroni}

docker/README.md

@@ -19,102 +19,97 @@ The haproxy listens on ports 5000 (connects to the primary) and 5001 (does load-
 
 Example session:
 
-    $ docker-compose up -d
+    $ docker compose up -d
-    Creating demo-haproxy ...
+    ✔ Network patroni_demo     Created
-    Creating demo-patroni2 ...
+    ✔ Container demo-etcd1     Started
-    Creating demo-patroni1 ...
+    ✔ Container demo-haproxy   Started
-    Creating demo-patroni3 ...
+    ✔ Container demo-patroni1  Started
-    Creating demo-etcd2 ...
+    ✔ Container demo-patroni2  Started
-    Creating demo-etcd1 ...
+    ✔ Container demo-patroni3  Started
-    Creating demo-etcd3 ...
+    ✔ Container demo-etcd2     Started
-    Creating demo-haproxy
+    ✔ Container demo-etcd3     Started
-    Creating demo-patroni2
-    Creating demo-patroni1
-    Creating demo-patroni3
-    Creating demo-etcd1
-    Creating demo-etcd2
-    Creating demo-etcd2 ... done
 
     $ docker ps
-    CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                              NAMES
+    CONTAINER ID   IMAGE     COMMAND                  CREATED          STATUS          PORTS                                                           NAMES
-    5b7a90b4cfbf        patroni             "/bin/sh /entrypoint…"   29 seconds ago      Up 27 seconds                                          demo-etcd2
+    a37bcec56726   patroni   "/bin/sh /entrypoint…"   15 minutes ago   Up 15 minutes                                                                   demo-etcd3
-    e30eea5222f2        patroni             "/bin/sh /entrypoint…"   29 seconds ago      Up 27 seconds                                          demo-etcd1
+    034ab73868a8   patroni   "/bin/sh /entrypoint…"   15 minutes ago   Up 15 minutes                                                                   demo-patroni2
-    83bcf3cb208f        patroni             "/bin/sh /entrypoint…"   29 seconds ago      Up 27 seconds                                          demo-etcd3
+    03837736f710   patroni   "/bin/sh /entrypoint…"   15 minutes ago   Up 15 minutes                                                                   demo-patroni3
-    922532c56e7d        patroni             "/bin/sh /entrypoint…"   29 seconds ago      Up 28 seconds                                          demo-patroni3
+    22815c3d85b3   patroni   "/bin/sh /entrypoint…"   15 minutes ago   Up 15 minutes                                                                   demo-etcd2
-    14f875e445f3        patroni             "/bin/sh /entrypoint…"   29 seconds ago      Up 28 seconds                                          demo-patroni2
+    814b4304d132   patroni   "/bin/sh /entrypoint…"   15 minutes ago   Up 15 minutes   0.0.0.0:5000-5001->5000-5001/tcp, :::5000-5001->5000-5001/tcp   demo-haproxy
-    110d1073b383        patroni             "/bin/sh /entrypoint…"   29 seconds ago      Up 28 seconds                                          demo-patroni1
+    6375b0ba2d0a   patroni   "/bin/sh /entrypoint…"   15 minutes ago   Up 15 minutes                                                                   demo-patroni1
-    5af5e6e36028        patroni             "/bin/sh /entrypoint…"   29 seconds ago      Up 28 seconds       0.0.0.0:5000-5001->5000-5001/tcp   demo-haproxy
+    aef8bf3ee91f   patroni   "/bin/sh /entrypoint…"   15 minutes ago   Up 15 minutes                                                                   demo-etcd1
 
     $ docker logs demo-patroni1
-    2019-02-20 08:19:32,714 INFO: Failed to import patroni.dcs.consul
+    2023-11-21 09:04:33,547 INFO: Selected new etcd server http://172.29.0.3:2379
-    2019-02-20 08:19:32,737 INFO: Selected new etcd server http://etcd3:2379
+    2023-11-21 09:04:33,605 INFO: Lock owner: None; I am patroni1
-    2019-02-20 08:19:35,140 INFO: Lock owner: None; I am patroni1
+    2023-11-21 09:04:33,693 INFO: trying to bootstrap a new cluster
-    2019-02-20 08:19:35,174 INFO: trying to bootstrap a new cluster
     ...
-    2019-02-20 08:19:39,310 INFO: postmaster pid=37
+    2023-11-21 09:04:34.920 UTC [43] LOG:  starting PostgreSQL 15.5 (Debian 15.5-1.pgdg120+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14) 12.2.0, 64-bit
-    2019-02-20 08:19:39.314 UTC [37] LOG:  listening on IPv4 address "0.0.0.0", port 5432
+    2023-11-21 09:04:34.921 UTC [43] LOG:  listening on IPv4 address "0.0.0.0", port 5432
-    2019-02-20 08:19:39.321 UTC [37] LOG:  listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432"
+    2023-11-21 09:04:34,922 INFO: postmaster pid=43
-    2019-02-20 08:19:39.353 UTC [39] LOG:  database system was shut down at 2019-02-20 08:19:36 UTC
+    2023-11-21 09:04:34.922 UTC [43] LOG:  listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432"
-    2019-02-20 08:19:39.354 UTC [40] FATAL:  the database system is starting up
+    2023-11-21 09:04:34.925 UTC [47] LOG:  database system was shut down at 2023-11-21 09:04:34 UTC
-    localhost:5432 - rejecting connections
+    2023-11-21 09:04:34.928 UTC [43] LOG:  database system is ready to accept connections
-    2019-02-20 08:19:39.369 UTC [37] LOG:  database system is ready to accept connections
     localhost:5432 - accepting connections
-    2019-02-20 08:19:39,383 INFO: establishing a new patroni connection to the postgres cluster
+    localhost:5432 - accepting connections
-    2019-02-20 08:19:39,408 INFO: running post_bootstrap
+    2023-11-21 09:04:34,938 INFO: establishing a new patroni heartbeat connection to postgres
-    2019-02-20 08:19:39,432 WARNING: Could not activate Linux watchdog device: "Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog'"
+    2023-11-21 09:04:34,992 INFO: running post_bootstrap
-    2019-02-20 08:19:39,515 INFO: initialized a new cluster
+    2023-11-21 09:04:35,004 WARNING: User creation via "bootstrap.users" will be removed in v4.0.0
-    2019-02-20 08:19:49,424 INFO: Lock owner: patroni1; I am patroni1
+    2023-11-21 09:04:35,009 WARNING: Could not activate Linux watchdog device: Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog'
-    2019-02-20 08:19:49,447 INFO: Lock owner: patroni1; I am patroni1
+    2023-11-21 09:04:35,189 INFO: initialized a new cluster
-    2019-02-20 08:19:49,480 INFO: no action.  i am the leader with the lock
+    2023-11-21 09:04:35,328 INFO: no action. I am (patroni1), the leader with the lock
-    2019-02-20 08:19:59,422 INFO: Lock owner: patroni1; I am patroni1
+    2023-11-21 09:04:43,824 INFO: establishing a new patroni restapi connection to postgres
+    2023-11-21 09:04:45,322 INFO: no action. I am (patroni1), the leader with the lock
+    2023-11-21 09:04:55,320 INFO: no action. I am (patroni1), the leader with the lock
+    ...
 
     $ docker exec -ti demo-patroni1 bash
     postgres@patroni1:~$ patronictl list
-    +---------+----------+------------+--------+---------+----+-----------+
+    + Cluster: demo (7303838734793224214) --------+----+-----------+
-    | Cluster |  Member  |    Host    |  Role  |  State  | TL | Lag in MB |
+    | Member   | Host       | Role    | State     | TL | Lag in MB |
-    +---------+----------+------------+--------+---------+----+-----------+
+    +----------+------------+---------+-----------+----+-----------+
-    |   demo  | patroni1 | 172.22.0.3 | Leader | running |  1 |         0 |
+    | patroni1 | 172.29.0.2 | Leader  | running   |  1 |           |
-    |   demo  | patroni2 | 172.22.0.7 |        | running |  1 |         0 |
+    | patroni2 | 172.29.0.6 | Replica | streaming |  1 |         0 |
-    |   demo  | patroni3 | 172.22.0.4 |        | running |  1 |         0 |
+    | patroni3 | 172.29.0.5 | Replica | streaming |  1 |         0 |
-    +---------+----------+------------+--------+---------+----+-----------+
+    +----------+------------+---------+-----------+----+-----------+
 
-    postgres@patroni1:~$ etcdctl ls --recursive --sort -p /service/demo
+    postgres@patroni1:~$ etcdctl get --keys-only --prefix /service/demo
     /service/demo/config
     /service/demo/initialize
     /service/demo/leader
-    /service/demo/members/
     /service/demo/members/patroni1
     /service/demo/members/patroni2
     /service/demo/members/patroni3
-    /service/demo/optime/
+    /service/demo/status
-    /service/demo/optime/leader
 
     postgres@patroni1:~$ etcdctl member list
-    1bab629f01fa9065: name=etcd3 peerURLs=http://etcd3:2380 clientURLs=http://etcd3:2379 isLeader=false
+    2bf3e2ceda5d5960, started, etcd2, http://etcd2:2380, http://172.29.0.3:2379
-    8ecb6af518d241cc: name=etcd2 peerURLs=http://etcd2:2380 clientURLs=http://etcd2:2379 isLeader=true
+    55b3264e129c7005, started, etcd3, http://etcd3:2380, http://172.29.0.7:2379
-    b2e169fcb8a34028: name=etcd1 peerURLs=http://etcd1:2380 clientURLs=http://etcd1:2379 isLeader=false
+    acce7233f8ec127e, started, etcd1, http://etcd1:2380, http://172.29.0.8:2379
+
+
     postgres@patroni1:~$ exit
 
     $ docker exec -ti demo-haproxy bash
     postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -W
     Password: postgres
-    psql (11.2 (Ubuntu 11.2-1.pgdg18.04+1), server 10.7 (Debian 10.7-1.pgdg90+1))
+    psql (15.5 (Debian 15.5-1.pgdg120+1))
     Type "help" for help.
 
-    localhost/postgres=# select pg_is_in_recovery();
+    postgres=# SELECT pg_is_in_recovery();
      pg_is_in_recovery
     ───────────────────
      f
     (1 row)
 
-    localhost/postgres=# \q
+    postgres=# \q
 
-    $postgres@haproxy:~ psql -h localhost -p 5001 -U postgres -W
+    postgres@haproxy:~$ psql -h localhost -p 5001 -U postgres -W
     Password: postgres
-    psql (11.2 (Ubuntu 11.2-1.pgdg18.04+1), server 10.7 (Debian 10.7-1.pgdg90+1))
+    psql (15.5 (Debian 15.5-1.pgdg120+1))
     Type "help" for help.
 
-    localhost/postgres=# select pg_is_in_recovery();
+    postgres=# SELECT pg_is_in_recovery();
      pg_is_in_recovery
     ───────────────────
      t
@@ -127,81 +122,86 @@ The haproxy listens on ports 5000 (connects to the coordinator primary) and 5001
 
 Example session:
 
-    $ docker-compose -f docker-compose-citus.yml up -d
+    $ docker compose -f docker-compose-citus.yml up -d
-    Creating demo-work2-1 ... done
+    ✔ Network patroni_demo    Created
-    Creating demo-work1-1 ... done
+    ✔ Container demo-coord2   Started
-    Creating demo-etcd2   ... done
+    ✔ Container demo-work2-2  Started
-    Creating demo-etcd1   ... done
+    ✔ Container demo-etcd1    Started
-    Creating demo-coord3  ... done
+    ✔ Container demo-haproxy  Started
-    Creating demo-etcd3   ... done
+    ✔ Container demo-work1-1  Started
-    Creating demo-coord1  ... done
+    ✔ Container demo-work2-1  Started
-    Creating demo-haproxy ... done
+    ✔ Container demo-work1-2  Started
-    Creating demo-work2-2 ... done
+    ✔ Container demo-coord1   Started
-    Creating demo-coord2  ... done
+    ✔ Container demo-etcd3    Started
-    Creating demo-work1-2 ... done
+    ✔ Container demo-coord3   Started
+    ✔ Container demo-etcd2    Started
+
 
     $ docker ps
-    CONTAINER ID   IMAGE                  COMMAND                  CREATED         STATUS         PORTS                              NAMES
+    CONTAINER ID   IMAGE           COMMAND                  CREATED          STATUS          PORTS                                                           NAMES
-    852d8885a612   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-coord3
+    79c95492fac9   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-etcd3
-    cdd692f947ab   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-work1-2
+    77eb82d0f0c1   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-work2-1
-    9f4e340b36da   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-etcd3
+    03dacd7267ef   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-etcd1
-    d69c129a960a   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds                                      demo-etcd1
+    db9206c66f85   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-etcd2
-    c5849689b8cd   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds                                      demo-coord1
+    9a0fef7b7dd4   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-work1-2
-    c9d72bd6217d   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-work2-1
+    f06b031d99dc   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-work2-2
-    24b1b43efa05   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds                                      demo-coord2
+    f7c58545f314   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-coord2
-    cb0cc2b4ca0a   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-work2-2
+    383f9e7e188a   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-work1-1
-    9796c6b8aad5   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 5 seconds                                      demo-work1-1
+    f02e96dcc9d6   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-coord3
-    8baccd74dcae   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds                                      demo-etcd2
+    6945834b7056   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-coord1
-    353ec62a0187   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds   0.0.0.0:5000-5001->5000-5001/tcp   demo-haproxy
+    b96ca42f785d   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes   0.0.0.0:5000-5001->5000-5001/tcp, :::5000-5001->5000-5001/tcp   demo-haproxy
+
 
     $ docker logs demo-coord1
-    2023-01-05 15:09:31,295 INFO: Selected new etcd server http://172.27.0.4:2379
+    2023-11-21 09:36:14,293 INFO: Selected new etcd server http://172.30.0.4:2379
-    2023-01-05 15:09:31,388 INFO: Lock owner: None; I am coord1
+    2023-11-21 09:36:14,390 INFO: Lock owner: None; I am coord1
-    2023-01-05 15:09:31,501 INFO: trying to bootstrap a new cluster
+    2023-11-21 09:36:14,478 INFO: trying to bootstrap a new cluster
     ...
-    2023-01-05 15:09:45,096 INFO: postmaster pid=39
+    2023-11-21 09:36:16,475 INFO: postmaster pid=52
     localhost:5432 - no response
-    2023-01-05 15:09:45.137 UTC [39] LOG:  starting PostgreSQL 15.1 (Debian 15.1-1.pgdg110+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 10.2.1-6) 10.2.1 20210110, 64-bit
+    2023-11-21 09:36:16.495 UTC [52] LOG:  starting PostgreSQL 15.5 (Debian 15.5-1.pgdg120+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14) 12.2.0, 64-bit
-    2023-01-05 15:09:45.137 UTC [39] LOG:  listening on IPv4 address "0.0.0.0", port 5432
+    2023-11-21 09:36:16.495 UTC [52] LOG:  listening on IPv4 address "0.0.0.0", port 5432
-    2023-01-05 15:09:45.152 UTC [39] LOG:  listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432"
+    2023-11-21 09:36:16.496 UTC [52] LOG:  listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432"
-    2023-01-05 15:09:45.177 UTC [43] LOG:  database system was shut down at 2023-01-05 15:09:32 UTC
+    2023-11-21 09:36:16.498 UTC [56] LOG:  database system was shut down at 2023-11-21 09:36:15 UTC
-    2023-01-05 15:09:45.193 UTC [39] LOG:  database system is ready to accept connections
+    2023-11-21 09:36:16.501 UTC [52] LOG:  database system is ready to accept connections
     localhost:5432 - accepting connections
     localhost:5432 - accepting connections
-    2023-01-05 15:09:46,139 INFO: establishing a new patroni connection to the postgres cluster
+    2023-11-21 09:36:17,509 INFO: establishing a new patroni heartbeat connection to postgres
-    2023-01-05 15:09:46,208 INFO: running post_bootstrap
+    2023-11-21 09:36:17,569 INFO: running post_bootstrap
-    2023-01-05 15:09:47.209 UTC [55] LOG:  starting maintenance daemon on database 16386 user 10
+    2023-11-21 09:36:17,593 WARNING: User creation via "bootstrap.users" will be removed in v4.0.0
-    2023-01-05 15:09:47.209 UTC [55] CONTEXT:  Citus maintenance daemon for database 16386 user 10
+    2023-11-21 09:36:17,783 INFO: establishing a new patroni restapi connection to postgres
-    2023-01-05 15:09:47,215 WARNING: Could not activate Linux watchdog device: "Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog'"
+    2023-11-21 09:36:17,969 WARNING: Could not activate Linux watchdog device: Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog'
-    2023-01-05 15:09:47.446 UTC [41] LOG:  checkpoint starting: immediate force wait
+    2023-11-21 09:36:17.969 UTC [70] LOG:  starting maintenance daemon on database 16386 user 10
-    2023-01-05 15:09:47,466 INFO: initialized a new cluster
+    2023-11-21 09:36:17.969 UTC [70] CONTEXT:  Citus maintenance daemon for database 16386 user 10
-    2023-01-05 15:09:47,594 DEBUG: query(SELECT nodeid, groupid, nodename, nodeport, noderole FROM pg_catalog.pg_dist_node WHERE noderole = 'primary', ())
+    2023-11-21 09:36:18.159 UTC [54] LOG:  checkpoint starting: immediate force wait
-    2023-01-05 15:09:47,594 INFO: establishing a new patroni connection to the postgres cluster
+    2023-11-21 09:36:18,162 INFO: initialized a new cluster
-    2023-01-05 15:09:47,467 INFO: Lock owner: coord1; I am coord1
+    2023-11-21 09:36:18,164 INFO: Lock owner: coord1; I am coord1
-    2023-01-05 15:09:47,613 DEBUG: query(SELECT pg_catalog.citus_set_coordinator_host(%s, %s, 'primary', 'default'), ('172.27.0.6', 5432))
+    2023-11-21 09:36:18,297 INFO: Enabled synchronous replication
-    2023-01-05 15:09:47,924 INFO: no action. I am (coord1), the leader with the lock
+    2023-11-21 09:36:18,298 DEBUG: Adding the new task: PgDistNode(nodeid=None,group=0,host=172.30.0.3,port=5432,event=after_promote)
-    2023-01-05 15:09:51.282 UTC [41] LOG:  checkpoint complete: wrote 1086 buffers (53.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.029 s, sync=3.746 s, total=3.837 s; sync files=280, longest=0.028 s, average=0.014 s; distance=8965 kB, estimate=8965 kB
+    2023-11-21 09:36:18,298 DEBUG: Adding the new task: PgDistNode(nodeid=None,group=1,host=172.30.0.7,port=5432,event=after_promote)
-    2023-01-05 15:09:51.283 UTC [41] LOG:  checkpoint starting: immediate force wait
+    2023-11-21 09:36:18,298 DEBUG: Adding the new task: PgDistNode(nodeid=None,group=2,host=172.30.0.8,port=5432,event=after_promote)
-    2023-01-05 15:09:51.495 UTC [41] LOG:  checkpoint complete: wrote 18 buffers (0.9%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.044 s, sync=0.091 s, total=0.212 s; sync files=15, longest=0.015 s, average=0.007 s; distance=67 kB, estimate=8076 kB
+    2023-11-21 09:36:18,299 DEBUG: query(SELECT nodeid, groupid, nodename, nodeport, noderole FROM pg_catalog.pg_dist_node WHERE noderole = 'primary', ())
-    2023-01-05 15:09:57,467 INFO: Lock owner: coord1; I am coord1
+    2023-11-21 09:36:18,299 INFO: establishing a new patroni citus connection to postgres
-    2023-01-05 15:09:57,569 INFO: Assigning synchronous standby status to ['coord3']
+    2023-11-21 09:36:18,323 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.30.0.7', 5432, 1))
+    2023-11-21 09:36:18,361 INFO: no action. I am (coord1), the leader with the lock
+    2023-11-21 09:36:18,393 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.30.0.8', 5432, 2))
+    2023-11-21 09:36:28,164 INFO: Lock owner: coord1; I am coord1
+    2023-11-21 09:36:28,251 INFO: Assigning synchronous standby status to ['coord3']
     server signaled
-    2023-01-05 15:09:57.574 UTC [39] LOG:  received SIGHUP, reloading configuration files
+    2023-11-21 09:36:28.435 UTC [52] LOG:  received SIGHUP, reloading configuration files
-    2023-01-05 15:09:57.580 UTC [39] LOG:  parameter "synchronous_standby_names" changed to "coord3"
+    2023-11-21 09:36:28.436 UTC [52] LOG:  parameter "synchronous_standby_names" changed to "coord3"
-    2023-01-05 15:09:59,637 INFO: Synchronous standby status assigned to ['coord3']
+    2023-11-21 09:36:28.641 UTC [83] LOG:  standby "coord3" is now a synchronous standby with priority 1
-    2023-01-05 15:09:59,638 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.27.0.2', 5432, 1))
+    2023-11-21 09:36:28.641 UTC [83] STATEMENT:  START_REPLICATION SLOT "coord3" 0/3000000 TIMELINE 1
-    2023-01-05 15:09:59.690 UTC [67] LOG:  standby "coord3" is now a synchronous standby with priority 1
+    2023-11-21 09:36:30,582 INFO: Synchronous standby status assigned to ['coord3']
-    2023-01-05 15:09:59.690 UTC [67] STATEMENT:  START_REPLICATION SLOT "coord3" 0/3000000 TIMELINE 1
+    2023-11-21 09:36:30,626 INFO: no action. I am (coord1), the leader with the lock
-    2023-01-05 15:09:59,694 INFO: no action. I am (coord1), the leader with the lock
+    2023-11-21 09:36:38,250 INFO: no action. I am (coord1), the leader with the lock
-    2023-01-05 15:09:59,704 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.27.0.8', 5432, 2))
+    ...
-    2023-01-05 15:10:07,625 INFO: no action. I am (coord1), the leader with the lock
-    2023-01-05 15:10:17,579 INFO: no action. I am (coord1), the leader with the lock
 
     $ docker exec -ti demo-haproxy bash
     postgres@haproxy:~$ etcdctl member list
-    1bab629f01fa9065, started, etcd3, http://etcd3:2380, http://172.27.0.10:2379
+    2b28411e74c0c281, started, etcd3, http://etcd3:2380, http://172.30.0.4:2379
-    8ecb6af518d241cc, started, etcd2, http://etcd2:2380, http://172.27.0.4:2379
+    6c70137d27cfa6c1, started, etcd2, http://etcd2:2380, http://172.30.0.5:2379
-    b2e169fcb8a34028, started, etcd1, http://etcd1:2380, http://172.27.0.7:2379
+    a28f9a70ebf21304, started, etcd1, http://etcd1:2380, http://172.30.0.6:2379
 
     postgres@haproxy:~$ etcdctl get --keys-only --prefix /service/demo
     /service/demo/0/config
@@ -229,7 +229,7 @@ Example session:
 
     postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -d citus
     Password for user postgres: postgres
-    psql (15.1 (Debian 15.1-1.pgdg110+1))
+    psql (15.5 (Debian 15.5-1.pgdg120+1))
     SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, compression: off)
     Type "help" for help.
 
@@ -240,67 +240,67 @@ Example session:
     (1 row)
 
     citus=# table pg_dist_node;
-     nodeid | groupid |  nodename  | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards
+     nodeid | groupid |  nodename  | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards 
     --------+---------+------------+----------+----------+-------------+----------+----------+-------------+----------------+------------------
-          1 |       0 | 172.27.0.6 |     5432 | default  | t           | t        | primary  | default     | t              | f
+          1 |       0 | 172.30.0.3 |     5432 | default  | t           | t        | primary  | default     | t              | f
-          2 |       1 | 172.27.0.2 |     5432 | default  | t           | t        | primary  | default     | t              | t
+          2 |       1 | 172.30.0.7 |     5432 | default  | t           | t        | primary  | default     | t              | t
-          3 |       2 | 172.27.0.8 |     5432 | default  | t           | t        | primary  | default     | t              | t
+          3 |       2 | 172.30.0.8 |     5432 | default  | t           | t        | primary  | default     | t              | t
     (3 rows)
 
     citus=# \q
 
     postgres@haproxy:~$ patronictl list
-    + Citus cluster: demo ----------+--------------+---------+----+-----------+
+    + Citus cluster: demo ----------+--------------+-----------+----+-----------+
-    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
+    | Group | Member  | Host        | Role         | State     | TL | Lag in MB |
-    +-------+---------+-------------+--------------+---------+----+-----------+
+    +-------+---------+-------------+--------------+-----------+----+-----------+
-    |     0 | coord1  | 172.27.0.6  | Leader       | running |  1 |           |
+    |     0 | coord1  | 172.30.0.3  | Leader       | running   |  1 |           |
-    |     0 | coord2  | 172.27.0.5  | Replica      | running |  1 |         0 |
+    |     0 | coord2  | 172.30.0.12 | Replica      | streaming |  1 |         0 |
-    |     0 | coord3  | 172.27.0.9  | Sync Standby | running |  1 |         0 |
+    |     0 | coord3  | 172.30.0.2  | Sync Standby | streaming |  1 |         0 |
-    |     1 | work1-1 | 172.27.0.2  | Leader       | running |  1 |           |
+    |     1 | work1-1 | 172.30.0.7  | Leader       | running   |  1 |           |
-    |     1 | work1-2 | 172.27.0.12 | Sync Standby | running |  1 |         0 |
+    |     1 | work1-2 | 172.30.0.10 | Sync Standby | streaming |  1 |         0 |
-    |     2 | work2-1 | 172.27.0.11 | Sync Standby | running |  1 |         0 |
+    |     2 | work2-1 | 172.30.0.8  | Leader       | running   |  1 |           |
-    |     2 | work2-2 | 172.27.0.8  | Leader       | running |  1 |           |
+    |     2 | work2-2 | 172.30.0.11 | Sync Standby | streaming |  1 |         0 |
-    +-------+---------+-------------+--------------+---------+----+-----------+
+    +-------+---------+-------------+--------------+-----------+----+-----------+
+
 
     postgres@haproxy:~$ patronictl switchover --group 2 --force
     Current cluster topology
-    + Citus cluster: demo (group: 2, 7185185529556963355) +-----------+
+    + Citus cluster: demo (group: 2, 7303846899271086103) --+-----------+
-    | Member  | Host        | Role         | State   | TL | Lag in MB |
+    | Member  | Host        | Role         | State     | TL | Lag in MB |
-    +---------+-------------+--------------+---------+----+-----------+
+    +---------+-------------+--------------+-----------+----+-----------+
-    | work2-1 | 172.27.0.11 | Sync Standby | running |  1 |         0 |
+    | work2-1 | 172.30.0.8  | Leader       | running   |  1 |           |
-    | work2-2 | 172.27.0.8  | Leader       | running |  1 |           |
+    | work2-2 | 172.30.0.11 | Sync Standby | streaming |  1 |         0 |
-    +---------+-------------+--------------+---------+----+-----------+
+    +---------+-------------+--------------+-----------+----+-----------+
-    2023-01-05 15:29:29.54204 Successfully switched over to "work2-1"
+    2023-11-21 09:44:15.83849 Successfully switched over to "work2-2"
-    + Citus cluster: demo (group: 2, 7185185529556963355) -------+
+    + Citus cluster: demo (group: 2, 7303846899271086103) -------+
     | Member  | Host        | Role    | State   | TL | Lag in MB |
     +---------+-------------+---------+---------+----+-----------+
-    | work2-1 | 172.27.0.11 | Leader  | running |  1 |           |
+    | work2-1 | 172.30.0.8  | Replica | stopped |    |   unknown |
-    | work2-2 | 172.27.0.8  | Replica | stopped |    |   unknown |
+    | work2-2 | 172.30.0.11 | Leader  | running |  1 |           |
     +---------+-------------+---------+---------+----+-----------+
 
     postgres@haproxy:~$ patronictl list
-    + Citus cluster: demo ----------+--------------+---------+----+-----------+
+    + Citus cluster: demo ----------+--------------+-----------+----+-----------+
-    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
+    | Group | Member  | Host        | Role         | State     | TL | Lag in MB |
-    +-------+---------+-------------+--------------+---------+----+-----------+
+    +-------+---------+-------------+--------------+-----------+----+-----------+
-    |     0 | coord1  | 172.27.0.6  | Leader       | running |  1 |           |
+    |     0 | coord1  | 172.30.0.3  | Leader       | running   |  1 |           |
-    |     0 | coord2  | 172.27.0.5  | Replica      | running |  1 |         0 |
+    |     0 | coord2  | 172.30.0.12 | Replica      | streaming |  1 |         0 |
-    |     0 | coord3  | 172.27.0.9  | Sync Standby | running |  1 |         0 |
+    |     0 | coord3  | 172.30.0.2  | Sync Standby | streaming |  1 |         0 |
-    |     1 | work1-1 | 172.27.0.2  | Leader       | running |  1 |           |
+    |     1 | work1-1 | 172.30.0.7  | Leader       | running   |  1 |           |
-    |     1 | work1-2 | 172.27.0.12 | Sync Standby | running |  1 |         0 |
+    |     1 | work1-2 | 172.30.0.10 | Sync Standby | streaming |  1 |         0 |
-    |     2 | work2-1 | 172.27.0.11 | Leader       | running |  2 |           |
+    |     2 | work2-1 | 172.30.0.8  | Sync Standby | streaming |  2 |         0 |
-    |     2 | work2-2 | 172.27.0.8  | Sync Standby | running |  2 |         0 |
+    |     2 | work2-2 | 172.30.0.11 | Leader       | running   |  2 |           |
-    +-------+---------+-------------+--------------+---------+----+-----------+
+    +-------+---------+-------------+--------------+-----------+----+-----------+
 
     postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -d citus
-    Password for user postgres: postgres
+    psql (15.5 (Debian 15.5-1.pgdg120+1))
-    psql (15.1 (Debian 15.1-1.pgdg110+1))
     SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, compression: off)
     Type "help" for help.
 
     citus=# table pg_dist_node;
-     nodeid | groupid |  nodename   | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards
+     nodeid | groupid |  nodename   | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards 
     --------+---------+-------------+----------+----------+-------------+----------+----------+-------------+----------------+------------------
-          1 |       0 | 172.27.0.6  |     5432 | default  | t           | t        | primary  | default     | t              | f
+          1 |       0 | 172.30.0.3  |     5432 | default  | t           | t        | primary  | default     | t              | f
-          3 |       2 | 172.27.0.11 |     5432 | default  | t           | t        | primary  | default     | t              | t
+          3 |       2 | 172.30.0.11 |     5432 | default  | t           | t        | primary  | default     | t              | t
-          2 |       1 | 172.27.0.2  |     5432 | default  | t           | t        | primary  | default     | t              | t
+          2 |       1 | 172.30.0.7  |     5432 | default  | t           | t        | primary  | default     | t              | t
     (3 rows)

docker/entrypoint.sh

@@ -13,6 +13,8 @@ readonly PATRONI_NAMESPACE="${PATRONI_NAMESPACE%/}"
 DOCKER_IP=$(hostname --ip-address)
 readonly DOCKER_IP
 
+export DUMB_INIT_SETSID=0
+
 case "$1" in
     haproxy)
         haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D
@@ -72,4 +74,4 @@ export PATRONI_SUPERUSER_SSLKEY="${PATRONI_SUPERUSER_SSLKEY:-$PGSSLKEY}"
 export PATRONI_SUPERUSER_SSLCERT="${PATRONI_SUPERUSER_SSLCERT:-$PGSSLCERT}"
 export PATRONI_SUPERUSER_SSLROOTCERT="${PATRONI_SUPERUSER_SSLROOTCERT:-$PGSSLROOTCERT}"
 
-exec python3 /patroni.py postgres0.yml
+exec dumb-init python3 /patroni.py postgres0.yml

docs/CONTRIBUTING.rst

@@ -1,182 +1,12 @@
 .. _contributing:
 
-Contributing guidelines
+Contributing
-=======================
+============
 
-Wanna contribute to Patroni? Yay - here is how!
+Resources and information for developers can be found in the pages below.
 
-Chatting
+.. toctree::
---------
+   :maxdepth: 2
 
-Just want to chat with other Patroni users?  Looking for interactive troubleshooting help? Join us on channel `#patroni <https://postgresteam.slack.com/archives/C9XPYG92A>`__ in the `PostgreSQL Slack <https://pgtreats.info/slack-invite>`__.
+   contributing_guidelines
-
+   Patroni API docs<modules/modules>
-Running tests
--------------
-
-Requirements for running behave tests:
-
-1. PostgreSQL packages need to be installed.
-2. PostgreSQL binaries must be available in your `PATH`. You may need to add them to the path with something like `PATH=/usr/lib/postgresql/11/bin:$PATH python -m behave`.
-3. If you'd like to test with external DCSs (e.g., Etcd, Consul, and Zookeeper) you'll need the packages installed and respective services running and accepting unencrypted/unprotected connections on localhost and default port. In the case of Etcd or Consul, the behave test suite could start them up if binaries are available in the `PATH`.
-
-Install dependencies:
-
-.. code-block:: bash
-
-    # You may want to use Virtualenv or specify pip3.
-    pip install -r requirements.txt
-    pip install -r requirements.dev.txt
-
-After you have all dependencies installed, you can run the various test suites:
-
-.. code-block:: bash
-
-    # You may want to use Virtualenv or specify python3.
-
-    # Run flake8 to check syntax and formatting:
-    python setup.py flake8
-
-    # Run the pytest suite in tests/:
-    python setup.py test
-
-    # Run the behave (https://behave.readthedocs.io/en/latest/) test suite in features/;
-    # modify DCS as desired (raft has no dependencies so is the easiest to start with):
-    DCS=raft python -m behave
-
-Testing with tox
-----------------
-
-To run tox tests you only need to install one dependency (other than Python)
-
-.. code-block:: bash
-
-    pip install tox>=4
-
-If you wish to run `behave` tests then you also need docker installed.
-
-Tox configuration in `tox.ini` has "environments" to run the following tasks:
-
-* lint: Python code lint with `flake8`
-* test: unit tests for all available python interpreters with `pytest`,
-  generates XML reports or HTML reports if a TTY is detected
-* dep: detect package dependency conflicts using `pipdeptree`
-* type: static type checking with `pyright`
-* black: code formatting with `black`
-* docker-build: build docker image used for the `behave` env
-* docker-cmd: run arbitrary command with the above image
-* docker-behave-etcd: run tox for behave tests with above image
-* py*behave: run behave with available python interpreters (without docker, although
-  this is what is called inside docker containers)
-* docs: build docs with `sphinx`
-
-Running tox
-^^^^^^^^^^^
-
-To run the default env list; dep, lint, test, and docs, just run:
-
-.. code-block:: bash
-
-    tox
-
-The `test` envs can be run with the label `test`:
-
-.. code-block:: bash
-
-   tox -m test
-
-The `behave` docker tests can be run with the label `behave`:
-
-.. code-block:: bash
-
-   tox -m behave
-
-Similarly, docs has the label `docs`.
-
-All other envs can be run with their respective env names:
-
-.. code-block:: bash
-
-   tox -e lint
-   tox -e py39-test-lin
-
-It is also possible to select partial env lists using `factors`. For example, if you want to run
-all envs for python 3.10:
-
-.. code-block:: bash
-
-    tox -f py310
-
-This is equivalent to running all the envs listed below:
-
-.. code-block:: bash
-
-    $ tox -l -f py310
-    py310-test-lin
-    py310-test-mac
-    py310-test-win
-    py310-type-lin
-    py310-type-mac
-    py310-type-win
-    py310-behave-etcd-lin
-    py310-behave-etcd-win
-    py310-behave-etcd-mac
-
-
-You can list all configured combinations of environments with tox (>=v4) like so
-
-.. code-block:: bash
-
-    tox l
-
-The envs `test` and `docs` will attempt to open the HTML output files
-when the job completes, if tox is run with an active terminal. This
-is intended to be for benefit of the developer running this env locally.
-It will attempt to run `open` on a mac and `xdg-open` on Linux.
-To use a different command set the env var `OPEN_CMD` to the name or path of
-the command. If this step fails it will not fail the run overall.
-If you want to disable this facility set the env var `OPEN_CMD` to the `:` no-op command.
-
-.. code-block:: bash
-
-   OPEN_CMD=: tox -m docs
-
-Behave tests
-^^^^^^^^^^^^
-
-Behave tests with `-m behave` will build docker images based on PG_MAJOR version 11 through 15 and then run all
-behave tests. This can take quite a long time to run so you might want to limit the scope to a select version of
-Postgres or to a specific feature set or steps.
-
-To specify the version of postgres include the full name of the dependent image build env that you want and then the
-behave env name. For instance if you want Postgres 15 use:
-
-.. code-block:: bash
-
-    tox -e pg14-docker-build,pg14-docker-behave-etcd-lin
-
-If on the other hand you want to test a specific feature you can pass positional arguments to behave. This will run
-the watchdog behave feature test scenario with all versions of Postgres.
-
-.. code-block:: bash
-
-    tox -m behave -- features/watchdog.feature
-
-Of course you can combine the two.
-
-Reporting issues
-----------------
-
-If you have a question about patroni or have a problem using it, please read the :ref:`README <readme>` before filing an issue.
-Also double check with the current issues on our `Issues Tracker <https://github.com/zalando/patroni/issues>`__.
-
-Contributing a pull request
----------------------------
-
-1) Submit a comment to the relevant issue or create a new issue describing your proposed change.
-2) Do a fork, develop and test your code changes.
-3) Include documentation
-4) Submit a pull request.
-
-You'll get feedback about your pull request as soon as possible.
-
-Happy Patroni hacking ;-)

docs/ENVIRONMENT.rst

@@ -14,25 +14,24 @@ Global/Universal
 
 Log
 ---
+-  **PATRONI\_LOG\_TYPE**: sets the format of logs. Can be either **plain** or **json**. To use **json** format, you must have the :ref:`jsonlogger <extras>` installed. The default value is **plain**.
 -  **PATRONI\_LOG\_LEVEL**: sets the general logging level. Default value is **INFO** (see `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>`_)
 -  **PATRONI\_LOG\_TRACEBACK\_LEVEL**: sets the level where tracebacks will be visible. Default value is **ERROR**. Set it to **DEBUG** if you want to see tracebacks only if you enable **PATRONI\_LOG\_LEVEL=DEBUG**.
--  **PATRONI\_LOG\_FORMAT**: sets the log formatting string. Default value is **%(asctime)s %(levelname)s: %(message)s** (see `the LogRecord attributes <https://docs.python.org/3.6/library/logging.html#logrecord-attributes>`_)
+-  **PATRONI\_LOG\_FORMAT**: sets the log formatting string. If the log type is **plain**, the log format should be a string.
+   Refer to `the LogRecord attributes <https://docs.python.org/3.6/library/logging.html#logrecord-attributes>`_ for
+   available attributes. If the log type is **json**, the log format can be a list in addition to a string. Each list
+   item should correspond to LogRecord attributes. Be cautious that only the field name is required, and the **%(**
+   and **)** should be omitted. If you wish to print a log field with a different key name, use a dictionary where
+   the dictionary key is the log field, and the value is the name of the field you want to be printed in the log.
+   Default value is **%(asctime)s %(levelname)s: %(message)s**
 -  **PATRONI\_LOG\_DATEFORMAT**: sets the datetime formatting string. (see the `formatTime() documentation <https://docs.python.org/3.6/library/logging.html#logging.Formatter.formatTime>`_)
+-  **PATRONI\_LOG\_STATIC\_FIELDS**: add additional fields to the log. This option is only available when the log type is set to **json**. Example ``PATRONI_LOG_STATIC_FIELDS="{app: patroni}"``
 -  **PATRONI\_LOG\_MAX\_QUEUE\_SIZE**: Patroni is using two-step logging. Log records are written into the in-memory queue and there is a separate thread which pulls them from the queue and writes to stderr or file. The maximum size of the internal queue is limited by default by **1000** records, which is enough to keep logs for the past 1h20m.
 -  **PATRONI\_LOG\_DIR**: Directory to write application logs to. The directory must exist and be writable by the user executing Patroni. If you set this env variable, the application will retain 4 25MB logs by default. You can tune those retention values with `PATRONI_LOG_FILE_NUM` and `PATRONI_LOG_FILE_SIZE` (see below).
 -  **PATRONI\_LOG\_FILE\_NUM**: The number of application logs to retain.
 -  **PATRONI\_LOG\_FILE\_SIZE**: Size of patroni.log file (in bytes) that triggers a log rolling.
 -  **PATRONI\_LOG\_LOGGERS**: Redefine logging level per python module. Example ``PATRONI_LOG_LOGGERS="{patroni.postmaster: WARNING, urllib3: DEBUG}"``
 
-Bootstrap configuration
------------------------
-It is possible to create new database users right after the successful initialization of a new cluster. This process is defined by the following variables:
-
--  **PATRONI\_<username>\_PASSWORD='<password>'**
--  **PATRONI\_<username>\_OPTIONS='list,of,options'**
-
-Example: defining ``PATRONI_admin_PASSWORD=strongpasswd`` and ``PATRONI_admin_OPTIONS='createrole,createdb'`` will cause creation of the user **admin** with the password **strongpasswd** that is allowed to create other users and databases.
-
 Citus
 -----
 Enables integration Patroni with `Citus <https://docs.citusdata.com>`__. If configured, Patroni will take care of registering Citus worker nodes on the coordinator. You can find more information about Citus support :ref:`here <citus>`.
@@ -94,6 +93,7 @@ ZooKeeper
 -  **PATRONI\_ZOOKEEPER\_KEY\_PASSWORD**: (optional) The client key password.
 -  **PATRONI\_ZOOKEEPER\_VERIFY**: (optional) Whether to verify certificate or not. Defaults to ``true``.
 -  **PATRONI\_ZOOKEEPER\_SET\_ACLS**: (optional) If set, configure Kazoo to apply a default ACL to each ZNode that it creates. ACLs will assume 'x509' schema and should be specified as a dictionary with the principal as the key and one or more permissions as a list in the value.  Permissions may be one of ``CREATE``, ``READ``, ``WRITE``, ``DELETE`` or ``ADMIN``.  For example, ``set_acls: {CN=principal1: [CREATE, READ], CN=principal2: [ALL]}``.
+-  **PATRONI\_ZOOKEEPER\_AUTH\_DATA**: (optional) Authentication credentials to use for the connection. Should be a dictionary in the form that `scheme` is the key and `credential` is the value. Defaults to empty dictionary.
 
 .. note::
     It is required to install ``kazoo>=2.6.0`` to support SSL.
@@ -112,7 +112,11 @@ Kubernetes
 -  **PATRONI\_KUBERNETES\_NAMESPACE**: (optional) Kubernetes namespace where the Patroni pod is running. Default value is `default`.
 -  **PATRONI\_KUBERNETES\_LABELS**: Labels in format ``{label1: value1, label2: value2}``. These labels will be used to find existing objects (Pods and either Endpoints or ConfigMaps) associated with the current cluster. Also Patroni will set them on every object (Endpoint or ConfigMap) it creates.
 -  **PATRONI\_KUBERNETES\_SCOPE\_LABEL**: (optional) name of the label containing cluster name. Default value is `cluster-name`.
--  **PATRONI\_KUBERNETES\_ROLE\_LABEL**: (optional) name of the label containing Postgres role (`master` or `replica`). Patroni will set this label on the pod it is running in. Default value is `role`.
+-  **PATRONI\_KUBERNETES\_ROLE\_LABEL**: (optional) name of the label containing role (master or replica or other custom value). Patroni will set this label on the pod it runs in. Default value is ``role``.
+-  **PATRONI\_KUBERNETES\_LEADER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is `master`. Default value is `master`.
+-  **PATRONI\_KUBERNETES\_FOLLOWER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is `replica`. Default value is `replica`.
+-  **PATRONI\_KUBERNETES\_STANDBY\_LEADER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is ``standby_leader``. Default value is ``master``.
+-  **PATRONI\_KUBERNETES\_TMP\_ROLE\_LABEL**: (optional) name of the temporary label containing role (master or replica). Value of this label will always use the default of corresponding role. Set only when necessary.
 -  **PATRONI\_KUBERNETES\_USE\_ENDPOINTS**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
 -  **PATRONI\_KUBERNETES\_POD\_IP**: (optional) IP address of the pod Patroni is running in. This value is required when `PATRONI_KUBERNETES_USE_ENDPOINTS` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
 -  **PATRONI\_KUBERNETES\_PORTS**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``PATRONI_KUBERNETES_PORTS='[{"name": "postgresql", "port": 5432}]'`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `PATRONI_KUBERNETES_USE_ENDPOINTS` is set.
@@ -196,10 +200,19 @@ REST API
 -  **PATRONI\_RESTAPI\_HTTPS\_EXTRA\_HEADERS**: (optional) HTTPS headers let the REST API server pass additional information with an HTTP response when TLS is enabled. This will also pass additional information set in ``http_extra_headers``.
 -  **PATRONI\_RESTAPI\_REQUEST\_QUEUE\_SIZE**: (optional): Sets request queue size for TCP socket used by Patroni REST API.  Once the queue is full, further requests get a "Connection denied" error. The default value is 5.
 
+.. warning::
+
+    - The ``PATRONI_RESTAPI_CONNECT_ADDRESS`` must be accessible from all nodes of a given Patroni cluster. Internally Patroni is using it during the leader race to find nodes with minimal replication lag.
+    - If you enabled client certificates validation (``PATRONI_RESTAPI_VERIFY_CLIENT`` is set to ``required``), you also **must** provide **valid client certificates** in the ``PATRONI_CTL_CERTFILE``, ``PATRONI_CTL_KEYFILE``, ``PATRONI_CTL_KEYFILE_PASSWORD``. If not provided, Patroni will not work correctly.
+
+
 CTL
 ---
--  **PATRONICTL\_CONFIG\_FILE**: location of the configuration file.
+-  **PATRONICTL\_CONFIG\_FILE**: (optional) location of the configuration file.
--  **PATRONI\_CTL\_INSECURE**: Allow connections to REST API without verifying SSL certs.
+-  **PATRONI\_CTL\_USERNAME**: (optional) Basic-auth username for accessing protected REST API endpoints. If not provided :ref:`patronictl` will use the value provided for REST API "username" parameter.
--  **PATRONI\_CTL\_CACERT**: Specifies the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs. If not provided patronictl will use the value provided for REST API "cafile" parameter.
+-  **PATRONI\_CTL\_PASSWORD**: (optional) Basic-auth password for accessing protected REST API endpoints. If not provided :ref:`patronictl` will use the value provided for REST API "password" parameter.
--  **PATRONI\_CTL\_CERTFILE**: Specifies the file with the client certificate in the PEM format. If not provided patronictl will use the value provided for REST API "certfile" parameter.
+-  **PATRONI\_CTL\_INSECURE**: (optional) Allow connections to REST API without verifying SSL certs.
--  **PATRONI\_CTL\_KEYFILE**: Specifies the file with the client secret key in the PEM format. If not provided patronictl will use the value provided for REST API "keyfile" parameter.
+-  **PATRONI\_CTL\_CACERT**: (optional) Specifies the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs. If not provided :ref:`patronictl` will use the value provided for REST API "cafile" parameter.
+-  **PATRONI\_CTL\_CERTFILE**: (optional) Specifies the file with the client certificate in the PEM format.
+-  **PATRONI\_CTL\_KEYFILE**: (optional) Specifies the file with the client secret key in the PEM format.
+-  **PATRONI\_CTL\_KEYFILE\_PASSWORD**: (optional) Specifies a password for decrypting the client keyfile.

docs/README.rst

@@ -25,83 +25,7 @@ We report new releases information :ref:`here <releases>`.
 Technical Requirements/Installation
 -----------------------------------
 
-**Pre-requirements for Mac OS**
+Go :ref:`here <installation>` for guidance on installing and upgrading Patroni on various platforms.
-
-To install requirements on a Mac, run the following:
-
-::
-
-    brew install postgresql etcd haproxy libyaml python
-
-.. _psycopg2_install_options:
-
-**Psycopg**
-
-Starting from `psycopg2-2.8 <http://initd.org/psycopg/articles/2019/04/04/psycopg-28-released/>`__ the binary version of psycopg2 will no longer be installed by default. Installing it from the source code requires C compiler and postgres+python dev packages.
-Since in the python world it is not possible to specify dependency as ``psycopg2 OR psycopg2-binary`` you will have to decide how to install it.
-
-There are a few options available:
-
-1. Use the package manager from your distro
-
-::
-
-    sudo apt-get install python-psycopg2   # install python2 psycopg2 module on Debian/Ubuntu
-    sudo apt-get install python3-psycopg2  # install python3 psycopg2 module on Debian/Ubuntu
-    sudo yum install python-psycopg2       # install python2 psycopg2 on RedHat/Fedora/CentOS
-
-2. Install psycopg2 from the binary package
-
-::
-
-    pip install psycopg2-binary
-
-3. Install psycopg2 from source
-
-::
-
-    pip install psycopg2>=2.5.4
-
-4. Use psycopg 3.0 instead of psycopg2
-
-::
-
-    pip install psycopg[binary]>=3.0.0
-
-**General installation for pip**
-
-Patroni can be installed with pip:
-
-::
-
-    pip install patroni[dependencies]
-
-where dependencies can be either empty, or consist of one or more of the following:
-
-etcd or etcd3
-    `python-etcd` module in order to use Etcd as Distributed Configuration Store (DCS)
-consul
-    `python-consul` module in order to use Consul as DCS
-zookeeper
-    `kazoo` module in order to use Zookeeper as DCS
-exhibitor
-    `kazoo` module in order to use Exhibitor as DCS (same dependencies as for Zookeeper)
-kubernetes
-    `kubernetes` module in order to use Kubernetes as DCS in Patroni
-raft
-    `pysyncobj` module in order to use python Raft implementation as DCS
-aws
-    `boto3` in order to use AWS callbacks
-
-For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is:
-
-::
-
-    pip install patroni[etcd,aws]
-
-Note that external tools to call in the replica creation or custom bootstrap scripts (i.e. WAL-E) should be installed
-independently of Patroni.
-
 
 .. _running_configuring:
 
@@ -165,10 +89,6 @@ Applications Should Not Use Superusers
 
 When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
 
-.. |Build Status| image:: https://travis-ci.org/zalando/patroni.svg?branch=master
-   :target: https://travis-ci.org/zalando/patroni
-.. |Coverage Status| image:: https://coveralls.io/repos/zalando/patroni/badge.svg?branch=master
-   :target: https://coveralls.io/r/zalando/patroni?branch=master
 
 Testing Your HA Solution
 --------------------------------------

docs/citus.rst

@@ -34,18 +34,24 @@ There are only a few simple rules you need to follow:
 
 After that you just need to start Patroni and it will handle the rest:
 
+0. Patroni will set ``bootstrap.dcs.synchronous_mode`` to :ref:`quorum <quorum_mode>`
+   if it is not explicitly set to any other value.
 1. ``citus`` extension will be automatically added to ``shared_preload_libraries``.
 2. If ``max_prepared_transactions`` isn't explicitly set in the global
    :ref:`dynamic configuration <dynamic_configuration>` Patroni will
    automatically set it to ``2*max_connections``.
-3. The ``citus.database`` will be automatically created followed by ``CREATE EXTENSION citus``.
+3. The ``citus.local_hostname`` GUC value will be adjusted from ``localhost`` to the
-4. Current superuser :ref:`credentials <postgresql_settings>` will be added to the ``pg_dist_authinfo``
+   value that Patroni is using in order to connect to the local PostgreSQL
+   instance. The value sometimes should be different from the ``localhost``
+   because PostgreSQL might be not listening on it.
+4. The ``citus.database`` will be automatically created followed by ``CREATE EXTENSION citus``.
+5. Current superuser :ref:`credentials <postgresql_settings>` will be added to the ``pg_dist_authinfo``
    table to allow cross-node communication. Don't forget to update them if
    later you decide to change superuser username/password/sslcert/sslkey!
-5. The coordinator primary node will automatically discover worker primary
+6. The coordinator primary node will automatically discover worker primary
    nodes and add them to the ``pg_dist_node`` table using the
    ``citus_add_node()`` function.
-6. Patroni will also maintain ``pg_dist_node`` in case failover/switchover
+7. Patroni will also maintain ``pg_dist_node`` in case failover/switchover
    on the coordinator or worker clusters occurs.
 
 patronictl
@@ -57,7 +63,7 @@ clusters that are just logically groupped together using the
 PostgreSQL. Therefore in most cases it is not possible to manage them as a
 single entity.
 
-It results in two major differences in ``patronictl`` behaviour when
+It results in two major differences in :ref:`patronictl` behaviour when
 ``patroni.yaml`` has the ``citus`` section comparing with the usual:
 
 1. The ``list`` and the ``topology`` by default output all members of the Citus
@@ -65,44 +71,44 @@ It results in two major differences in ``patronictl`` behaviour when
    which Citus group they belong to.
 2. For all ``patronictl`` commands the new option is introduced, named
    ``--group``. For some commands the default value for the group might be
-   taken from the ``patroni.yaml``. For example, ``patronictl pause`` will
+   taken from the ``patroni.yaml``. For example, :ref:`patronictl_pause` will
    enable the maintenance mode by default for the ``group`` that is set in the
-   ``citus`` section, but for example for ``patronictl  switchover`` or
+   ``citus`` section, but for example for :ref:`patronictl_switchover` or
-   ``patronictl remove`` the group must be explicitly specified.
+   :ref:`patronictl_remove` the group must be explicitly specified.
 
-An example of ``patronictl list`` output for the Citus cluster::
+An example of :ref:`patronictl_list` output for the Citus cluster::
 
     postgres@coord1:~$ patronictl list demo
-    + Citus cluster: demo ----------+--------------+---------+----+-----------+
+    + Citus cluster: demo ----------+----------------+---------+----+-----------+
-    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
+    | Group | Member  | Host        | Role           | State   | TL | Lag in MB |
-    +-------+---------+-------------+--------------+---------+----+-----------+
+    +-------+---------+-------------+----------------+---------+----+-----------+
-    |     0 | coord1  | 172.27.0.10 | Replica      | running |  1 |         0 |
+    |     0 | coord1  | 172.27.0.10 | Replica        | running |  1 |         0 |
-    |     0 | coord2  | 172.27.0.6  | Sync Standby | running |  1 |         0 |
+    |     0 | coord2  | 172.27.0.6  | Quorum Standby | running |  1 |         0 |
-    |     0 | coord3  | 172.27.0.4  | Leader       | running |  1 |           |
+    |     0 | coord3  | 172.27.0.4  | Leader         | running |  1 |           |
-    |     1 | work1-1 | 172.27.0.8  | Sync Standby | running |  1 |         0 |
+    |     1 | work1-1 | 172.27.0.8  | Quorum Standby | running |  1 |         0 |
-    |     1 | work1-2 | 172.27.0.2  | Leader       | running |  1 |           |
+    |     1 | work1-2 | 172.27.0.2  | Leader         | running |  1 |           |
-    |     2 | work2-1 | 172.27.0.5  | Sync Standby | running |  1 |         0 |
+    |     2 | work2-1 | 172.27.0.5  | Quorum Standby | running |  1 |         0 |
-    |     2 | work2-2 | 172.27.0.7  | Leader       | running |  1 |           |
+    |     2 | work2-2 | 172.27.0.7  | Leader         | running |  1 |           |
-    +-------+---------+-------------+--------------+---------+----+-----------+
+    +-------+---------+-------------+----------------+---------+----+-----------+
 
 If we add the ``--group`` option, the output will change to::
 
     postgres@coord1:~$ patronictl list demo --group 0
-    + Citus cluster: demo (group: 0, 7179854923829112860) -----------+
+    + Citus cluster: demo (group: 0, 7179854923829112860) -+-----------+
-    | Member | Host        | Role         | State   | TL | Lag in MB |
+    | Member | Host        | Role           | State   | TL | Lag in MB |
-    +--------+-------------+--------------+---------+----+-----------+
+    +--------+-------------+----------------+---------+----+-----------+
-    | coord1 | 172.27.0.10 | Replica      | running |  1 |         0 |
+    | coord1 | 172.27.0.10 | Replica        | running |  1 |         0 |
-    | coord2 | 172.27.0.6  | Sync Standby | running |  1 |         0 |
+    | coord2 | 172.27.0.6  | Quorum Standby | running |  1 |         0 |
-    | coord3 | 172.27.0.4  | Leader       | running |  1 |           |
+    | coord3 | 172.27.0.4  | Leader         | running |  1 |           |
-    +--------+-------------+--------------+---------+----+-----------+
+    +--------+-------------+----------------+---------+----+-----------+
 
     postgres@coord1:~$ patronictl list demo --group 1
-    + Citus cluster: demo (group: 1, 7179854923881963547) -----------+
+    + Citus cluster: demo (group: 1, 7179854923881963547) -+-----------+
-    | Member  | Host       | Role         | State   | TL | Lag in MB |
+    | Member  | Host       | Role           | State   | TL | Lag in MB |
-    +---------+------------+--------------+---------+----+-----------+
+    +---------+------------+----------------+---------+----+-----------+
-    | work1-1 | 172.27.0.8 | Sync Standby | running |  1 |         0 |
+    | work1-1 | 172.27.0.8 | Quorum Standby | running |  1 |         0 |
-    | work1-2 | 172.27.0.2 | Leader       | running |  1 |           |
+    | work1-2 | 172.27.0.2 | Leader         | running |  1 |           |
-    +---------+------------+--------------+---------+----+-----------+
+    +---------+------------+----------------+---------+----+-----------+
 
 Citus worker switchover
 -----------------------
@@ -115,31 +121,31 @@ the coordinator for the shards hosted on a worker node. The switchover then
 happens while the traffic is kept on the coordinator, and resumes as soon as a
 new primary worker node is ready to accept read-write queries.
 
-An example of ``patronictl switchover`` on the worker cluster::
+An example of :ref:`patronictl_switchover` on the worker cluster::
 
     postgres@coord1:~$ patronictl switchover demo
-    + Citus cluster: demo ----------+--------------+---------+----+-----------+
+    + Citus cluster: demo ----------+----------------+---------+----+-----------+
-    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
+    | Group | Member  | Host        | Role           | State   | TL | Lag in MB |
-    +-------+---------+-------------+--------------+---------+----+-----------+
+    +-------+---------+-------------+----------------+---------+----+-----------+
-    |     0 | coord1  | 172.27.0.10 | Replica      | running |  1 |         0 |
+    |     0 | coord1  | 172.27.0.10 | Replica        | running |  1 |         0 |
-    |     0 | coord2  | 172.27.0.6  | Sync Standby | running |  1 |         0 |
+    |     0 | coord2  | 172.27.0.6  | Quorum Standby | running |  1 |         0 |
-    |     0 | coord3  | 172.27.0.4  | Leader       | running |  1 |           |
+    |     0 | coord3  | 172.27.0.4  | Leader         | running |  1 |           |
-    |     1 | work1-1 | 172.27.0.8  | Leader       | running |  1 |           |
+    |     1 | work1-1 | 172.27.0.8  | Leader         | running |  1 |           |
-    |     1 | work1-2 | 172.27.0.2  | Sync Standby | running |  1 |         0 |
+    |     1 | work1-2 | 172.27.0.2  | Quorum Standby | running |  1 |         0 |
-    |     2 | work2-1 | 172.27.0.5  | Sync Standby | running |  1 |         0 |
+    |     2 | work2-1 | 172.27.0.5  | Quorum Standby | running |  1 |         0 |
-    |     2 | work2-2 | 172.27.0.7  | Leader       | running |  1 |           |
+    |     2 | work2-2 | 172.27.0.7  | Leader         | running |  1 |           |
-    +-------+---------+-------------+--------------+---------+----+-----------+
+    +-------+---------+-------------+----------------+---------+----+-----------+
     Citus group: 2
     Primary [work2-2]:
     Candidate ['work2-1'] []:
     When should the switchover take place (e.g. 2022-12-22T08:02 )  [now]:
     Current cluster topology
-    + Citus cluster: demo (group: 2, 7179854924063375386) -----------+
+    + Citus cluster: demo (group: 2, 7179854924063375386) -+-----------+
-    | Member  | Host       | Role         | State   | TL | Lag in MB |
+    | Member  | Host       | Role           | State   | TL | Lag in MB |
-    +---------+------------+--------------+---------+----+-----------+
+    +---------+------------+----------------+---------+----+-----------+
-    | work2-1 | 172.27.0.5 | Sync Standby | running |  1 |         0 |
+    | work2-1 | 172.27.0.5 | Quorum Standby | running |  1 |         0 |
-    | work2-2 | 172.27.0.7 | Leader       | running |  1 |           |
+    | work2-2 | 172.27.0.7 | Leader         | running |  1 |           |
-    +---------+------------+--------------+---------+----+-----------+
+    +---------+------------+----------------+---------+----+-----------+
     Are you sure you want to switchover cluster demo, demoting current primary work2-2? [y/N]: y
     2022-12-22 07:02:40.33003 Successfully switched over to "work2-1"
     + Citus cluster: demo (group: 2, 7179854924063375386) ------+
@@ -150,17 +156,17 @@ An example of ``patronictl switchover`` on the worker cluster::
     +---------+------------+---------+---------+----+-----------+
 
     postgres@coord1:~$ patronictl list demo
-    + Citus cluster: demo ----------+--------------+---------+----+-----------+
+    + Citus cluster: demo ----------+----------------+---------+----+-----------+
-    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
+    | Group | Member  | Host        | Role           | State   | TL | Lag in MB |
-    +-------+---------+-------------+--------------+---------+----+-----------+
+    +-------+---------+-------------+----------------+---------+----+-----------+
-    |     0 | coord1  | 172.27.0.10 | Replica      | running |  1 |         0 |
+    |     0 | coord1  | 172.27.0.10 | Replica        | running |  1 |         0 |
-    |     0 | coord2  | 172.27.0.6  | Sync Standby | running |  1 |         0 |
+    |     0 | coord2  | 172.27.0.6  | Quorum Standby | running |  1 |         0 |
-    |     0 | coord3  | 172.27.0.4  | Leader       | running |  1 |           |
+    |     0 | coord3  | 172.27.0.4  | Leader         | running |  1 |           |
-    |     1 | work1-1 | 172.27.0.8  | Leader       | running |  1 |           |
+    |     1 | work1-1 | 172.27.0.8  | Leader         | running |  1 |           |
-    |     1 | work1-2 | 172.27.0.2  | Sync Standby | running |  1 |         0 |
+    |     1 | work1-2 | 172.27.0.2  | Quorum Standby | running |  1 |         0 |
-    |     2 | work2-1 | 172.27.0.5  | Leader       | running |  2 |           |
+    |     2 | work2-1 | 172.27.0.5  | Leader         | running |  2 |           |
-    |     2 | work2-2 | 172.27.0.7  | Sync Standby | running |  2 |         0 |
+    |     2 | work2-2 | 172.27.0.7  | Quorum Standby | running |  2 |         0 |
-    +-------+---------+-------------+--------------+---------+----+-----------+
+    +-------+---------+-------------+----------------+---------+----+-----------+
 
 And this is how it looks on the coordinator side::
 
@@ -343,7 +349,7 @@ Citus upgrades and PostgreSQL major upgrades
 
 First, please read about upgrading Citus version in the `documentation`__.
 There is one minor change in the process. When executing upgrade, you have to
-use ``patronictl restart`` instead of ``systemctl restart`` to restart
+use :ref:`patronictl_restart` instead of ``systemctl restart`` to restart
 PostgreSQL.
 
 __ https://docs.citusdata.com/en/latest/admin_guide/upgrading_citus.html

docs/conf.py

@@ -20,10 +20,15 @@
 import os
 
 import sys
+
 sys.path.insert(0, os.path.abspath('..'))
 
 from patroni.version import __version__
 
+project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+module_dir = os.path.abspath(os.path.join(project_root, 'patroni'))
+excludes = ['tests', 'setup.py', 'conf']
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -33,11 +38,28 @@ from patroni.version import __version__
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['sphinx.ext.intersphinx',
+extensions = [
+    'sphinx.ext.intersphinx',
     'sphinx.ext.todo',
     'sphinx.ext.mathjax',
     'sphinx.ext.ifconfig',
-    'sphinx.ext.viewcode']
+    # 'sphinx.ext.viewcode',
+    'sphinx_github_style',  # Generate "View on GitHub" for source code
+    'sphinxcontrib.apidoc',  # For generating module docs from code
+    'sphinx.ext.autodoc',  # For generating module docs from docstrings
+    'sphinx.ext.napoleon',  # For Google and Numpy formatted docstrings
+]
+apidoc_module_dir = module_dir
+apidoc_output_dir = 'modules'
+apidoc_excluded_paths = excludes
+apidoc_separate_modules = True
+
+# Include autodoc for all members, including private ones and the ones that are missing a docstring.
+autodoc_default_options = {
+    "members": True,
+    "undoc-members": True,
+    "private-members": True,
+}
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -90,10 +112,10 @@ todo_include_todos = True
 # a list of builtin themes.
 #
 
+html_theme = 'sphinx_rtd_theme'
 on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
 if not on_rtd:  # only import and set the theme if we're building docs locally
     import sphinx_rtd_theme
-    html_theme = 'sphinx_rtd_theme'
     html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
@@ -107,6 +129,34 @@ if not on_rtd:  # only import and set the theme if we're building docs locally
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
+# Replace "source" links with "edit on GitHub" when using rtd theme
+html_context = {
+    'display_github': True,
+    'github_user': 'zalando',
+    'github_repo': 'patroni',
+    'github_version': 'master',
+    'conf_py_path': '/docs/',
+}
+
+# sphinx-github-style options, https://sphinx-github-style.readthedocs.io/en/latest/index.html
+
+# The name of the top-level package.
+top_level = "patroni"
+
+# The blob to link to on GitHub - any of "head", "last_tag", or "{blob}"
+# linkcode_blob = 'head'
+
+# The link to your GitHub repository formatted as https://github.com/user/repo
+# If not provided, will attempt to create the link from the html_context dict
+# linkcode_url = f"https://github.com/{html_context['github_user']}/" \
+#                f"{html_context['github_repo']}/{html_context['github_version']}"
+
+# The text to use for the linkcode link
+# linkcode_link_text: str = "View on GitHub"
+
+# A linkcode_resolve() function to use for resolving the link target
+# linkcode_resolve: types.FunctionType
+
 
 # -- Options for HTMLHelp output ------------------------------------------
 
@@ -165,7 +215,6 @@ texinfo_documents = [
 ]
 
 
-
 # -- Options for Epub output ----------------------------------------------
 
 # Bibliographic Dublin Core info.
@@ -187,10 +236,65 @@ epub_copyright = copyright
 epub_exclude_files = ['search.html']
 
 
-
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {'python': ('https://docs.python.org/', None)}
 
+
+# Remove these pages from index, references, toc trees, etc.
+# If the builder is not 'html' then add the API docs modules index to pages to be removed.
+exclude_from_builder = {
+    'latex': ['modules/modules'],
+    'epub': ['modules/modules'],
+}
+# Internal holding list, anything added here will always be excluded
+_docs_to_remove = []
+
+
+def builder_inited(app):
+    """Run during Sphinx `builder-inited` phase.
+
+    Set a config value to builder name and add module docs to `docs_to_remove`.
+    """
+    print(f'The builder is: {app.builder.name}')
+    app.add_config_value('builder', app.builder.name, 'env')
+
+    # Remove pages when builder matches any referenced in exclude_from_builder
+    if exclude_from_builder.get(app.builder.name):
+        _docs_to_remove.extend(exclude_from_builder[app.builder.name])
+
+
+def env_get_outdated(app, env, added, changed, removed):
+    """Run during Sphinx `env-get-outdated` phase.
+
+    Remove the items listed in `docs_to_remove` from known pages.
+    """
+    added.difference_update(_docs_to_remove)
+    changed.difference_update(_docs_to_remove)
+    removed.update(_docs_to_remove)
+    return []
+
+
+def doctree_read(app, doctree):
+    """Run during Sphinx `doctree-read` phase.
+
+    Remove the items listed in `docs_to_remove` from the table of contents.
+    """
+    from sphinx import addnodes
+    for toc_tree_node in doctree.traverse(addnodes.toctree):
+        for e in toc_tree_node['entries']:
+            ref = str(e[1])
+            if ref in _docs_to_remove:
+                toc_tree_node['entries'].remove(e)
+
+
+def autodoc_skip(app, what, name, obj, would_skip, options):
+    """Include autodoc of ``__init__`` methods, which are skipped by default."""
+    if name == "__init__":
+        return False
+    return would_skip
+
+
+
 # A possibility to have an own stylesheet, to add new rules or override existing ones
 # For the latter case, the CSS specificity of the rules should be higher than the default ones
 def setup(app):
@@ -198,3 +302,9 @@ def setup(app):
         app.add_css_file('custom.css')
     else:
         app.add_stylesheet('custom.css')
+
+    # Run extra steps to remove module docs when running with a non-html builder
+    app.connect('builder-inited', builder_inited)
+    app.connect('env-get-outdated', env_get_outdated)
+    app.connect('doctree-read', doctree_read)
+    app.connect("autodoc-skip-member", autodoc_skip)

docs/contributing_guidelines.rst

@@ -0,0 +1,190 @@
+.. _contributing_guidelines:
+
+Contributing guidelines
+=======================
+
+.. _chatting:
+
+Chatting
+--------
+
+If you have a question, looking for an interactive troubleshooting help or want to chat with other Patroni users, join us on channel `#patroni <https://postgresteam.slack.com/archives/C9XPYG92A>`__ in the `PostgreSQL Slack <https://pgtreats.info/slack-invite>`__.
+
+.. _reporting_bugs:
+
+Reporting bugs
+--------------
+
+Before reporting a bug please make sure to **reproduce it with the latest Patroni version**!
+Also please double check if the issue already exists in our `Issues Tracker <https://github.com/zalando/patroni/issues>`__.
+
+Running tests
+-------------
+
+Requirements for running behave tests:
+
+#. PostgreSQL packages including `contrib <https://www.postgresql.org/docs/current/contrib.html>`__ modules need to be installed.
+#. PostgreSQL binaries must be available in your `PATH`. You may need to add them to the path with something like `PATH=/usr/lib/postgresql/11/bin:$PATH python -m behave`.
+#. If you'd like to test with external DCSs (e.g., Etcd, Consul, and Zookeeper) you'll need the packages installed and respective services running and accepting unencrypted/unprotected connections on localhost and default port. In the case of Etcd or Consul, the behave test suite could start them up if binaries are available in the `PATH`.
+
+Install dependencies:
+
+.. code-block:: bash
+
+    # You may want to use Virtualenv or specify pip3.
+    pip install -r requirements.txt
+    pip install -r requirements.dev.txt
+
+After you have all dependencies installed, you can run the various test suites:
+
+.. code-block:: bash
+
+    # You may want to use Virtualenv or specify python3.
+
+    # Run flake8 to check syntax and formatting:
+    python setup.py flake8
+
+    # Run the pytest suite in tests/:
+    python setup.py test
+
+    # Moreover, you may want to run tests in different scopes for debugging purposes,
+    # the -s option include print output during test execution.
+    # Tests in pytest typically follow the pattern: FILEPATH::CLASSNAME::TESTNAME.
+    pytest -s tests/test_api.py
+    pytest -s tests/test_api.py::TestRestApiHandler
+    pytest -s tests/test_api.py::TestRestApiHandler::test_do_GET
+
+    # Run the behave (https://behave.readthedocs.io/en/latest/) test suite in features/;
+    # modify DCS as desired (raft has no dependencies so is the easiest to start with):
+    DCS=raft python -m behave
+
+Testing with tox
+----------------
+
+To run tox tests you only need to install one dependency (other than Python)
+
+.. code-block:: bash
+
+    pip install tox>=4
+
+If you wish to run `behave` tests then you also need docker installed.
+
+Tox configuration in `tox.ini` has "environments" to run the following tasks:
+
+* lint: Python code lint with `flake8`
+* test: unit tests for all available python interpreters with `pytest`,
+  generates XML reports or HTML reports if a TTY is detected
+* dep: detect package dependency conflicts using `pipdeptree`
+* type: static type checking with `pyright`
+* black: code formatting with `black`
+* docker-build: build docker image used for the `behave` env
+* docker-cmd: run arbitrary command with the above image
+* docker-behave-etcd: run tox for behave tests with above image
+* py*behave: run behave with available python interpreters (without docker, although
+  this is what is called inside docker containers)
+* docs: build docs with `sphinx`
+
+Running tox
+^^^^^^^^^^^
+
+To run the default env list; dep, lint, test, and docs, just run:
+
+.. code-block:: bash
+
+    tox
+
+The `test` envs can be run with the label `test`:
+
+.. code-block:: bash
+
+   tox -m test
+
+The `behave` docker tests can be run with the label `behave`:
+
+.. code-block:: bash
+
+   tox -m behave
+
+Similarly, docs has the label `docs`.
+
+All other envs can be run with their respective env names:
+
+.. code-block:: bash
+
+   tox -e lint
+   tox -e py39-test-lin
+
+It is also possible to select partial env lists using `factors`. For example, if you want to run
+all envs for python 3.10:
+
+.. code-block:: bash
+
+    tox -f py310
+
+This is equivalent to running all the envs listed below:
+
+.. code-block:: bash
+
+    $ tox -l -f py310
+    py310-test-lin
+    py310-test-mac
+    py310-test-win
+    py310-type-lin
+    py310-type-mac
+    py310-type-win
+    py310-behave-etcd-lin
+    py310-behave-etcd-win
+    py310-behave-etcd-mac
+
+
+You can list all configured combinations of environments with tox (>=v4) like so
+
+.. code-block:: bash
+
+    tox l
+
+The envs `test` and `docs` will attempt to open the HTML output files
+when the job completes, if tox is run with an active terminal. This
+is intended to be for benefit of the developer running this env locally.
+It will attempt to run `open` on a mac and `xdg-open` on Linux.
+To use a different command set the env var `OPEN_CMD` to the name or path of
+the command. If this step fails it will not fail the run overall.
+If you want to disable this facility set the env var `OPEN_CMD` to the `:` no-op command.
+
+.. code-block:: bash
+
+   OPEN_CMD=: tox -m docs
+
+Behave tests
+^^^^^^^^^^^^
+
+Behave tests with `-m behave` will build docker images based on PG_MAJOR version 11 through 16 and then run all
+behave tests. This can take quite a long time to run so you might want to limit the scope to a select version of
+Postgres or to a specific feature set or steps.
+
+To specify the version of postgres include the full name of the dependent image build env that you want and then the
+behave env name. For instance if you want Postgres 14 use:
+
+.. code-block:: bash
+
+    tox -e pg14-docker-build,pg14-docker-behave-etcd-lin
+
+If on the other hand you want to test a specific feature you can pass positional arguments to behave. This will run
+the watchdog behave feature test scenario with all versions of Postgres.
+
+.. code-block:: bash
+
+    tox -m behave -- features/watchdog.feature
+
+Of course you can combine the two.
+
+Contributing a pull request
+---------------------------
+
+#. Fork the repository, develop and test your code changes.
+#. Reflect changes in the user documentation.
+#. Submit a pull request with a clear description of the changes objective. Link an existing issue if necessary.
+
+You'll get feedback about your pull request as soon as possible.
+
+Happy Patroni hacking ;-)

docs/dcs_failsafe_mode.rst

@@ -60,4 +60,4 @@ F.A.Q.
 
 - How to enable the Failsafe Mode?
 
-  Before enabling the ``failsafe_mode`` please make sure that Patroni version on all members is up-to-date. After that, you can use either the ``PATCH /config`` :ref:`REST API <rest_api>` or ``patronictl edit-config -s failsafe_mode=true``
+  Before enabling the ``failsafe_mode`` please make sure that Patroni version on all members is up-to-date. After that, you can use either the ``PATCH /config`` :ref:`REST API <rest_api>` or :ref:`patronictl edit-config -s failsafe_mode=true <patronictl_edit_config_parameters>`

docs/dynamic_configuration.rst

@@ -6,17 +6,26 @@ Dynamic Configuration Settings
 
 Dynamic configuration is stored in the DCS (Distributed Configuration Store) and applied on all cluster nodes.
 
-In order to change the dynamic configuration you can use either ``patronictl edit-config`` tool or Patroni :ref:`REST API <rest_api>`.
+In order to change the dynamic configuration you can use either :ref:`patronictl_edit_config` tool or Patroni :ref:`REST API <rest_api>`.
+
+-  **loop\_wait**: the number of seconds the loop will sleep. Default value: 10, minimum possible value: 1
+-  **ttl**: the TTL to acquire the leader lock (in seconds). Think of it as the length of time before initiation of the automatic failover process. Default value: 30, minimum possible value: 20
+-  **retry\_timeout**: timeout for DCS and PostgreSQL operation retries (in seconds). DCS or network issues shorter than this will not cause Patroni to demote the leader. Default value: 10, minimum possible value: 3
+
+.. warning::
+    when changing values of **loop_wait**, **retry_timeout**, or **ttl** you have to follow the rule:
+
+    .. code-block:: python
+
+        loop_wait + 2 * retry_timeout <= ttl
+
 
--  **loop\_wait**: the number of seconds the loop will sleep. Default value: 10
--  **ttl**: the TTL to acquire the leader lock (in seconds). Think of it as the length of time before initiation of the automatic failover process. Default value: 30
--  **retry\_timeout**: timeout for DCS and PostgreSQL operation retries (in seconds). DCS or network issues shorter than this will not cause Patroni to demote the leader. Default value: 10
 -  **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election.
 -  **maximum\_lag\_on\_syncnode**: the maximum bytes a synchronous follower may lag before it is considered as an unhealthy candidate and swapped by healthy asynchronous follower. Patroni utilize the max replica lsn if there is more than one follower, otherwise it will use leader's current wal lsn. Default is -1, Patroni will not take action to swap synchronous unhealthy follower when the value is set to 0 or below. Please set the value high enough so Patroni won't swap synchrounous follower fequently during high transaction volume.
 -  **max\_timelines\_history**: maximum number of timeline history items kept in DCS.  Default value: 0. When set to 0, it keeps the full history in DCS.
 -  **primary\_start\_timeout**: the amount of time a primary is allowed to recover from failures before failover is triggered (in seconds). Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Worst case failover time for primary failure is: loop\_wait + primary\_start\_timeout + loop\_wait, unless primary\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff.
 -  **primary\_stop\_timeout**: The number of seconds Patroni is allowed to wait when stopping Postgres and effective only when synchronous_mode is enabled. When set to > 0 and the synchronous_mode is enabled, Patroni sends SIGKILL to the postmaster if the stop operation is running for more than the value set by primary\_stop\_timeout. Set the value according to your durability/availability tradeoff. If the parameter is not set or set <= 0, primary\_stop\_timeout does not apply.
--  **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that successfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See :ref:`replication modes documentation <replication_modes>` for details.
+-  **synchronous\_mode**: turns on synchronous replication mode. Possible values: ``off``, ``on``, ``quorum``. In this mode the leader takes care of management of ``synchronous_standby_names``, and only the last known leader, or one of synchronous replicas, are allowed to participate in leader race. Synchronous mode makes sure that successfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See :ref:`replication modes documentation <replication_modes>` for details.
 -  **synchronous\_mode\_strict**: prevents disabling synchronous replication if no synchronous replicas are available, blocking all client writes to the primary. See :ref:`replication modes documentation <replication_modes>` for details.
 -  **failsafe\_mode**: Enables :ref:`DCS Failsafe Mode <dcs_failsafe_mode>`. Defaults to `false`.
 -  **postgresql**:
@@ -46,9 +55,9 @@ In order to change the dynamic configuration you can use either ``patronictl edi
    -  **archive\_cleanup\_command**: cleanup command for standby leader
    -  **recovery\_min\_apply\_delay**: how long to wait before actually apply WAL records on a standby leader
 
--  **slots**: define permanent replication slots. These slots will be preserved during switchover/failover. The logical slots are copied from the primary to a standby with restart, and after that their position advanced every **loop_wait** seconds (if necessary). Copying logical slot files performed via ``libpq`` connection and using either rewind or superuser credentials (see **postgresql.authentication** section). There is always a chance that the logical slot position on the replica is a bit behind the former primary, therefore application should be prepared that some messages could be received the second time after the failover. The easiest way of doing so - tracking ``confirmed_flush_lsn``. Enabling permanent logical replication slots requires **postgresql.use_slots** to be set and will also automatically enable the ``hot_standby_feedback``. Since the failover of logical replication slots is unsafe on PostgreSQL 9.6 and older and PostgreSQL version 10 is missing some important functions, the feature only works with PostgreSQL 11+.
+-  **slots**: define permanent replication slots. These slots will be preserved during switchover/failover. Permanent slots that don't exist will be created by Patroni. With PostgreSQL 11 onwards permanent physical slots are created on all nodes and their position is advanced every **loop_wait** seconds. For PostgreSQL versions older than 11 permanent physical replication slots are maintained only on the current primary. The logical slots are copied from the primary to a standby with restart, and after that their position advanced every **loop_wait** seconds (if necessary). Copying logical slot files performed via ``libpq`` connection and using either rewind or superuser credentials (see **postgresql.authentication** section). There is always a chance that the logical slot position on the replica is a bit behind the former primary, therefore application should be prepared that some messages could be received the second time after the failover. The easiest way of doing so - tracking ``confirmed_flush_lsn``. Enabling permanent replication slots requires **postgresql.use_slots** to be set to ``true``. If there are permanent logical replication slots defined Patroni will automatically enable the ``hot_standby_feedback``. Since the failover of logical replication slots is unsafe on PostgreSQL 9.6 and older and PostgreSQL version 10 is missing some important functions, the feature only works with PostgreSQL 11+.
 
-   -  **my\_slot\_name**: the name of replication slot. If the permanent slot name matches with the name of the current primary it will not be created. Everything else is the responsibility of the operator to make sure that there are no clashes in names between replication slots automatically created by Patroni for members and permanent replication slots.
+   -  **my\_slot\_name**: the name of the permanent replication slot. If the permanent slot name matches with the name of the current node it will not be created on this node. If you add a permanent physical replication slot which name matches the name of a Patroni member, Patroni will ensure that the slot that was created is not removed even if the corresponding member becomes unresponsive, situation which would normally result in the slot's removal by Patroni. Although this can be useful in some situations, such as when you want replication slots used by members to persist during temporary failures or when importing existing members to a new Patroni cluster (see :ref:`Convert a Standalone to a Patroni Cluster <existing_data>` for details), caution should be exercised by the operator that these clashes in names are not persisted in the DCS, when the slot is no longer required, due to its effect on normal functioning of Patroni.
 
       -  **type**: slot type. Could be ``physical`` or ``logical``. If the slot is logical, you have to additionally define ``database`` and ``plugin``.
       -  **database**: the database name where logical slots should be created.
@@ -80,4 +89,26 @@ Note: **slots** is a hashmap while **ignore_slots** is an array. For example:
             plugin: test_decoding
           - name: ignored_physical_slot_name
             type: physical
-          ...
+          ...
+
+Note: if cluster topology is static (fixed number of nodes that never change their names) you can configure permanent physical replication slots with names corresponding to names of nodes to avoid recycling of WAL files while replica is temporary down:
+
+.. code:: YAML
+
+        slots:
+          node_name1:
+            type: physical
+          node_name2:
+            type: physical
+          node_name3:
+            type: physical
+          ...
+
+
+.. warning::
+   Permanent replication slots are synchronized only from the ``primary``/``standby_leader`` to replica nodes. That means, applications are supposed to be using them only from the leader node. Using them on replica nodes will cause indefinite growth of ``pg_wal`` on all other nodes in the cluster.
+   An exception to that rule are permanent physical slots that match the Patroni member names, if you happen to configure any. Those will be synchronized among all nodes as they are used for replication among them.
+
+
+.. warning::
+   Setting ``nostream`` tag on standby disables copying and synchronization of permanent logical replication slots on the node itself and all its cascading replicas if any.

docs/existing_data.rst

@@ -10,18 +10,58 @@ To deploy a Patroni cluster without using a pre-existing PostgreSQL instance, se
 Procedure
 ---------
 
-A Patroni cluster can be started with a data directory from a single-node PostgreSQL database. This is achieved by following closely these steps:
+You can find below an overview of steps for converting an existing Postgres cluster to a Patroni managed cluster. In the steps we assume all nodes that are part of the existing cluster are currently up and running, and that you *do not* intend to change Postgres configuration while the migration is ongoing. The steps:
 
-1. Manually start PostgreSQL daemon
+#. Create the Postgres users as explained for :ref:`authentication <postgresql_settings>` section of the Patroni configuration. You can find sample SQL commands to create the users in the code block below, in which you need to replace the usernames and passwords as per your environment. If you already have the relevant users, then you can skip this step.
-2. Create Patroni superuser and replication users as defined in the :ref:`authentication <postgresql_settings>` section of the Patroni configuration. If this user is created in SQL, the following queries achieve this:
 
-.. code-block:: sql
+   .. code-block:: sql
 
-   CREATE USER $PATRONI_SUPERUSER_USERNAME WITH SUPERUSER ENCRYPTED PASSWORD '$PATRONI_SUPERUSER_PASSWORD';
+      -- Patroni superuser
-   CREATE USER $PATRONI_REPLICATION_USERNAME WITH REPLICATION ENCRYPTED PASSWORD '$PATRONI_REPLICATION_PASSWORD';
+      -- Replace PATRONI_SUPERUSER_USERNAME and PATRONI_SUPERUSER_PASSWORD accordingly
+      CREATE USER PATRONI_SUPERUSER_USERNAME WITH SUPERUSER ENCRYPTED PASSWORD 'PATRONI_SUPERUSER_PASSWORD';
 
-3. Start Patroni (e.g. ``patroni /etc/patroni/patroni.yml``). It automatically detects that PostgreSQL daemon is already running but its configuration might be out-of-date.
+      -- Patroni replication user
-4. Ask Patroni to restart the node with ``patronictl restart cluster-name node-name``. This step is only required if PostgreSQL configuration is out-of-date.
+      -- Replace PATRONI_REPLICATION_USERNAME and PATRONI_REPLICATION_PASSWORD accordingly
+      CREATE USER PATRONI_REPLICATION_USERNAME WITH REPLICATION ENCRYPTED PASSWORD 'PATRONI_REPLICATION_PASSWORD';
+
+      -- Patroni rewind user, if you intend to enable use_pg_rewind in your Patroni configuration
+      -- Replace PATRONI_REWIND_USERNAME and PATRONI_REWIND_PASSWORD accordingly
+      CREATE USER PATRONI_REWIND_USERNAME WITH ENCRYPTED PASSWORD 'PATRONI_REWIND_PASSWORD';
+      GRANT EXECUTE ON function pg_catalog.pg_ls_dir(text, boolean, boolean) TO PATRONI_REWIND_USERNAME;
+      GRANT EXECUTE ON function pg_catalog.pg_stat_file(text, boolean) TO PATRONI_REWIND_USERNAME;
+      GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text) TO PATRONI_REWIND_USERNAME;
+      GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text, bigint, bigint, boolean) TO PATRONI_REWIND_USERNAME;
+
+#. Perform the following steps on all Postgres nodes. Perform all steps on one node before proceeding with the next node. Start with the primary node, then proceed with each standby node:
+
+   #. If you are running Postgres through systemd, then disable the Postgres systemd unit. This is performed as Patroni manages starting and stopping the Postgres daemon.
+
+   #. Create a YAML configuration file for Patroni. You can use :ref:`Patroni configuration generation and validation tooling <validate_generate_config>` for that.
+
+      * **Note (specific for the primary node):** If you have replication slots being used for replication between cluster members, then it is recommended that you enable ``use_slots`` and configure the existing replication slots as permanent via the ``slots`` configuration item. Be aware that Patroni automatically creates replication slots for replication between members, and drops replication slots that it does not recognize, when ``use_slots`` is enabled. The idea of using permanent slots here is to allow your existing slots to persist while the migration to Patroni is in progress. See :ref:`YAML Configuration Settings <yaml_configuration>` for details.
+
+   #. Start Patroni using the ``patroni`` systemd service unit. It automatically detects that Postgres is already running and starts monitoring the instance.
+
+#. Hand over Postgres "start up procedure" to Patroni. In order to do that you need to restart the cluster members through :ref:`patronictl restart cluster-name member-name <patronictl_restart_parameters>` command. For minimal downtime you might want to split this step into:
+
+   #. Immediate restart of the standby nodes.
+   #. Scheduled restart of the primary node within a maintenance window.
+
+#. If you configured permanent slots in step ``1.2.``, then you should remove them from ``slots`` configuration through :ref:`patronictl edit-config cluster-name member-name <patronictl_edit_config_parameters>` command once the ``restart_lsn`` of the slots created by Patroni is able to catch up with the ``restart_lsn`` of the original slots for the corresponding members. By removing the slots from ``slots`` configuration you will allow Patroni to drop the original slots from your cluster once they are not needed anymore. You can find below an example query to check the ``restart_lsn`` of a couple slots, so you can compare them:
+
+   .. code-block:: sql
+
+      -- Assume original_slot_for_member_x is the name of the slot in your original
+      -- cluster for replicating changes to member X, and slot_for_member_x is the
+      -- slot created by Patroni for that purpose. You need restart_lsn of
+      -- slot_for_member_x to be >= restart_lsn of original_slot_for_member_x
+      SELECT slot_name,
+             restart_lsn
+      FROM pg_replication_slots
+      WHERE slot_name IN (
+          'original_slot_for_member_x',
+          'slot_for_member_x'
+      )
 
 .. _major_upgrade:
 
@@ -30,14 +70,14 @@ Major Upgrade of PostgreSQL Version
 
 The only possible way to do a major upgrade currently is:
 
-1. Stop Patroni
+#. Stop Patroni
-2. Upgrade PostgreSQL binaries and perform `pg_upgrade <https://www.postgresql.org/docs/current/pgupgrade.html>`_ on the primary node
+#. Upgrade PostgreSQL binaries and perform `pg_upgrade <https://www.postgresql.org/docs/current/pgupgrade.html>`_ on the primary node
-3. Update patroni.yml
+#. Update patroni.yml
-4. Remove the initialize key from DCS or wipe complete cluster state from DCS. The second one could be achieved by running ``patronictl remove <cluster-name>``. It is necessary because pg_upgrade runs initdb which actually creates a new database with a new PostgreSQL system identifier.
+#. Remove the initialize key from DCS or wipe complete cluster state from DCS. The second one could be achieved by running :ref:`patronictl remove cluster-name <patronictl_remove_parameters>` . It is necessary because pg_upgrade runs initdb which actually creates a new database with a new PostgreSQL system identifier.
-5. If you wiped the cluster state in the previous step, you may wish to copy patroni.dynamic.json from old data dir to the new one.  It will help you to retain some PostgreSQL parameters you had set before.
+#. If you wiped the cluster state in the previous step, you may wish to copy patroni.dynamic.json from old data dir to the new one.  It will help you to retain some PostgreSQL parameters you had set before.
-6. Start Patroni on the primary node.
+#. Start Patroni on the primary node.
-7. Upgrade PostgreSQL binaries, update patroni.yml and wipe the data_dir on standby nodes.
+#. Upgrade PostgreSQL binaries, update patroni.yml and wipe the data_dir on standby nodes.
-8. Start Patroni on the standby nodes and wait for the replication to complete.
+#. Start Patroni on the standby nodes and wait for the replication to complete.
 
 Running pg_upgrade on standby nodes is not supported by PostgreSQL. If you know what you are doing, you can try the rsync procedure described in https://www.postgresql.org/docs/current/pgupgrade.html instead of wiping data_dir on standby nodes. The safest way is however to let Patroni replicate the data for you.
 

docs/faq.rst

@@ -0,0 +1,329 @@
+.. _faq:
+
+FAQ
+===
+
+In this section you will find answers for the most frequently asked questions about Patroni.
+Each sub-section attempts to focus on different kinds of questions.
+
+We hope that this helps you to clarify most of your questions.
+If you still have further concerns or find yourself facing an unexpected issue, please refer to :ref:`chatting` and :ref:`reporting_bugs` for instructions on how to get help or report issues.
+
+Comparison with other HA solutions
+----------------------------------
+
+Why does Patroni require a separate cluster of DCS nodes while other solutions like ``repmgr`` do not?
+    There are different ways of implementing HA solutions, each of them with their pros and cons.
+
+    Software like ``repmgr`` performs communication among the nodes to decide when actions should be taken.
+
+    Patroni on the other hand relies on the state stored in the DCS. The DCS acts as a source of truth for Patroni to decide what it should do.
+
+    While having a separate DCS cluster can make you bloat your architecture, this approach also makes it less likely for split-brain scenarios to happen in your Postgres cluster.
+
+What is the difference between Patroni and other HA solutions in regards to Postgres management?
+    Patroni does not just manage the high availability of the Postgres cluster but also manages Postgres itself.
+
+    If Postgres nodes do not exist yet, it takes care of bootstrapping the primary and the standby nodes, and also manages Postgres configuration of the nodes. If the Postgres nodes already exist, Patroni will take over management of the cluster.
+
+    Besides the above, Patroni also has self-healing capabilities. In other words, if a primary node fails, Patroni will not only fail over to a replica, but also attempt to rejoin the former primary as a replica of the new primary. Similarly, if a replica fails, Patroni will attempt to rejoin that replica.
+
+    That is way we call Patroni as a "template for HA solutions". It goes further than just managing physical replication: it manages Postgres as a whole.
+
+DCS
+---
+
+Can I use the same ``etcd`` cluster to store data from two or more Patroni clusters?
+    Yes, you can!
+
+    Information about a Patroni cluster is stored in the DCS under a path prefixed with the ``namespace`` and ``scope`` Patroni settings.
+
+    As long as you do not have conflicting namespace and scope across different Patroni clusters, you should be able to use the same DCS cluster to store information from multiple Patroni clusters.
+
+What occurs if I attempt to use the same combination of ``namespace`` and ``scope`` for different Patroni clusters that point to the same DCS cluster?
+    The second Patroni cluster that attempts to use the same ``namespace`` and ``scope`` will not be able to manage Postgres because it will find information related with that same combination in the DCS, but with an incompatible Postgres system identifier.
+    The mismatch on the system identifier causes Patroni to abort the management of the second cluster, as it assumes that refers to a different cluster and that the user has misconfigured Patroni.
+
+    Make sure to use different ``namespace`` / ``scope`` when dealing with different Patroni clusters that share the same DCS cluster.
+
+What occurs if I lose my DCS cluster?
+    The DCS is used to store basically status and the dynamic configuration of the Patroni cluster.
+
+    They very first consequence is that all the Patroni clusters that rely on that DCS will go to read-only mode -- unless :ref:`dcs_failsafe_mode` is enabled.
+
+What should I do if I lose my DCS cluster?
+    There are three possible outcomes upon losing your DCS cluster:
+
+    1. The DCS cluster is fully recovered: this requires no action from the Patroni side. Once the DCS cluster is recovered, Patroni should be able to recover too;
+    2. The DCS cluster is re-created in place, and the endpoints remain the same. No changes are required on the Patroni side;
+    3. A new DCS cluster is created with different endpoints. You will need to update the DCS endpoints in the Patroni configuration of each Patroni node.
+
+    If you face scenario ``2.`` or ``3.`` Patroni will take care of creating the status information again based on the current status of the cluster, and recreate the dynamic configuration on the DCS based on a backup file named ``patroni.dynamic.json`` which is stored inside the Postgres data directory of each member of the Patroni cluster.
+
+What occurs if I lose majority in my DCS cluster?
+    The DCS will become unresponsive, which will cause Patroni to demote the current read/write Postgres node.
+
+    Remember: Patroni relies on the state of the DCS to take actions on the cluster.
+
+    You can use the :ref:`dcs_failsafe_mode` to alleviate that situation.
+
+patronictl
+----------
+
+Do I need to run :ref:`patronictl` in the Patroni host?
+    No, you do not need to do that.
+
+    Running :ref:`patronictl` in the Patroni host is handy if you have access to the Patroni host because you can use the very same configuration file from the ``patroni`` agent for the :ref:`patronictl` application.
+
+    However, :ref:`patronictl` is basically a client and it can be executed from remote machines. You just need to provide it with enough configuration so it can reach the DCS and the REST API of the Patroni member(s).
+
+Why did the information from one of my Patroni members disappear from the output of :ref:`patronictl_list` command?
+    Information shown by :ref:`patronictl_list` is based on the contents of the DCS.
+
+    If information about a member disappeared from the DCS it is very likely that the Patroni agent on that node is not running anymore, or it is not able to communicate with the DCS.
+
+    As the member is not able to update the information, the information eventually expires from the DCS, and consequently the member is not shown anymore in the output of :ref:`patronictl_list`.
+
+Why is the information about one of my Patroni members not up-to-date in the output of :ref:`patronictl_list` command?
+    Information shown by :ref:`patronictl_list` is based on the contents of the DCS.
+
+    By default, that information is updated by Patroni roughly every ``loop_wait`` seconds.
+    In other words, even if everything is normally functional you may still see a "delay" of up to ``loop_wait`` seconds in the information stored in the DCS.
+
+    Be aware that that is not a rule, though. Some operations performed by Patroni cause it to immediately update the DCS information.
+
+Configuration
+-------------
+
+What is the difference between dynamic configuration and local configuration?
+    Dynamic configuration (or global configuration) is the configuration stored in the DCS, and which is applied to all members of the Patroni cluster.
+    This is primarily where you should store your configuration.
+
+    Settings that are specific to a node, or settings that you would like to overwrite the global configuration with, you should set only on the desired Patroni member as a local configuration.
+    That local configuration can be specified either through the configuration file or through environment variables.
+
+    See more in :ref:`patroni_configuration`.
+
+What are the types of configuration in Patroni, and what is the precedence?
+    The types are:
+
+    * Dynamic configuration: applied to all members;
+    * Local configuration: applied to the local member, overrides dynamic configuration;
+    * Environment configuration: applied to the local member, overrides both dynamic and local configuration.
+
+    **Note:** some Postgres GUCs can only be set globally, i.e., through dynamic configuration. Besides that, there are GUCs which Patroni enforces a hard-coded value.
+
+    See more in :ref:`patroni_configuration`.
+
+Is there any facility to help me create my Patroni configuration file?
+    Yes, there is.
+
+    You can use ``patroni --generate-sample-config`` or ``patroni --generate-config`` commands to generate a sample Patroni configuration or a Patroni configuration based on an existing Postgres instance, respectively.
+
+    Please refer to :ref:`generate_sample_config` and :ref:`generate_config` for more details.
+
+I changed my parameters under ``bootstrap.dcs`` configuration but Patroni is not applying the changes to the cluster members. What is wrong?
+    The values configured under ``bootstrap.dcs`` are only used when bootstrapping a fresh cluster. Those values will be written to the DCS during the bootstrap.
+
+    After the bootstrap phase finishes, you will only be able to change the dynamic configuration through the DCS.
+
+    Refer to the next question for more details.
+
+How can I change my dynamic configuration?
+    You need to change the configuration in the DCS. That is accomplished either through:
+
+    * :ref:`patronictl_edit_config`; or
+    * A ``PATCH`` request to :ref:`config_endpoint`.
+
+How can I change my local configuration?
+    You need to change the configuration file of the corresponding Patroni member and signal the Patroni agent with ``SIHGUP``. You can do that using either of these approaches:
+
+    * Send a ``POST`` request to the REST API :ref:`reload_endpoint`; or
+    * Run :ref:`patronictl_reload`; or
+    * Locally signal the Patroni process with ``SIGHUP``:
+
+        * If you started Patroni through systemd, you can use the command ``systemctl reload PATRONI_UNIT.service``, ``PATRONI_UNIT`` being the name of the Patroni service; or
+        * If you started Patroni through other means, you will need to identify the ``patroni`` process and run ``kill -s HUP PID``, ``PID`` being the process ID of the ``patroni`` process.
+
+    **Note:** there are cases where a reload through the :ref:`patronictl_reload` may not work:
+
+    * Expired REST API certificates: you can mitigate that by using the ``-k`` option of the :ref:`patronictl`;
+    * Wrong credentials: for example when changing ``restapi`` or ``ctl`` credentials in the configuration file, and using that same configuration file for Patroni and :ref:`patronictl`.
+
+How can I change my environment configuration?
+    The environment configuration is only read by Patroni during startup.
+
+    With that in mind, if you change the environment configuration you will need to restart the corresponding Patroni agent.
+
+    Take care to not cause a failover in the cluster! You might be interested in checking :ref:`patronictl_pause`.
+
+What occurs if I change a Postgres GUC that requires a reload?
+    When you change the dynamic or the local configuration as explained in the previous questions, Patroni will take care of reloading the Postgres configuration for you.
+
+What occurs if I change a Postgres GUC that requires a restart?
+    Patroni will mark the affected members with a flag of ``pending restart``.
+
+    It is up to you to determine when and how to restart the members. That can be accomplished either through:
+
+    * :ref:`patronictl_restart`; or
+    * A ``POST`` request to :ref:`restart_endpoint`.
+
+    **Note:** some Postgres GUCs require a special management in terms of the order for restarting the Postgres nodes. Refer to :ref:`shared_memory_gucs` for more details.
+
+What is the difference between ``etcd`` and ``etcd3`` in Patroni configuration?
+    ``etcd`` uses the API version 2 of ``etcd``, while ``etcd3`` uses the API version 3 of ``etcd``.
+
+    Be aware that information stored by the API version 2 is not manageable by API version 3 and vice-versa.
+
+    We recommend that you configure ``etcd3`` instead of ``etcd`` because:
+
+    * API version 2 is disabled by default from Etcd v3.4 onward;
+    * API version 2 will be completely removed on Etcd v3.6.
+
+I have ``use_slots`` enabled in my Patroni configuration, but when a cluster member goes offline for some time, the replication slot used by that member is dropped on the upstream node. What can I do to avoid that issue?
+    You can configure a permanent physical replication slot for the members.
+
+    Since Patroni ``3.2.0`` it is now possible to have member slots as permanent slots managed by Patroni.
+
+    Patroni will create the permanent physical slots on all nodes, and make sure to not remove the slots, as well as to advance the slots' LSN on all nodes according to the LSN that has been consumed by the member.
+
+    Later, if you decide to remove the corresponding member, it's **your responsability** to adjust the permanent slots configuration, otherwise Patroni will keep the slots around forever.
+
+    **Note:** on Patroni older than ``3.2.0`` you could still have member slots configured as permanent physical slots, however they would be managed only on the current leader. That is, in case of failover/switchover these slots would be created on the new leader, but that wouldn't guarantee that it had all WAL segments for the absent node.
+
+    **Note:** even with Patroni ``3.2.0`` there might be a small race condition. In the very beginning, when the slot is created on the replica it could be ahead of the same slot on the leader and in case if nobody is consuming the slot there is still a chance that some files could be missing after failover. With that in mind, it is recommended that you configure continuous archiving, which makes it possible to restore required WALs or perform PITR.
+
+What is the difference between ``loop_wait``, ``retry_timeout`` and ``ttl``?
+    Patroni performs what we call a HA cycle from time to time. On each HA cycle it takes care of performing a series of checks on the cluster to determine its healthiness, and depending on the status it may take actions, like failing over to a standby.
+
+    ``loop_wait`` determines for how long, in seconds, Patroni should sleep before performing a new cycle of HA checks.
+
+    ``retry_timeout`` sets the timeout for retry operations on the DCS and on Postgres. For example: if the DCS is unresponsive for more than ``retry_timeout`` seconds, Patroni might demote the primary node as a security action.
+
+    ``ttl`` sets the lease time on the ``leader`` lock in the DCS. If the current leader of the cluster is not able to renew the lease during its HA cycles for longer than ``ttl``, then the lease will expire and that will trigger a ``leader race`` in the cluster.
+
+    **Note:** when modifying these settings, please keep in mind that Patroni enforces the rule and minimal values described in :ref:`dynamic_configuration` section of the docs.
+
+Postgres management
+-------------------
+
+Can I change Postgres GUCs directly in Postgres configuration?
+    You can, but you should avoid that.
+
+    Postgres configuration is managed by Patroni, and attempts to edit the configuration files may end up being frustrated by Patroni as it may eventually overwrite them.
+
+    There are a few options available to overcome the management performed by Patroni:
+
+    * Change Postgres GUCs through ``$PGDATA/postgresql.base.conf``; or
+    * Define a ``postgresql.custom_conf`` which will be used instead of ``postgresql.base.conf`` so you can manage that externally; or
+    * Change GUCs using ``ALTER SYSTEM`` / ``ALTER DATABASE`` / ``ALTER USER``.
+
+    You can find more information about that in the section :ref:`important_configuration_rules`.
+
+    In any case we recommend that you manage all the Postgres configuration through Patroni. That will centralize the management and make it easier to debug Patroni when needed.
+
+Can I restart Postgres nodes directly?
+    No, you should **not** attempt to manage Postgres directly!
+
+    Any attempt of bouncing the Postgres server without Patroni can lead your cluster to face failovers.
+
+    If you need to manage the Postgres server, do that through the ways exposed by Patroni.
+
+Is Patroni able to take over management of an already existing Postgres cluster?
+    Yes, it can!
+
+    Please refer to :ref:`existing_data` for detailed instructions.
+
+How does Patroni manage Postgres?
+    Patroni takes care of bringing Postgres up and down by running the Postgres binaries, like ``pg_ctl`` and ``postgres``.
+
+    With that in mind you **MUST** disable any other sources that could manage the Postgres clusters, like the systemd units, e.g. ``postgresql.service``. Only Patroni should be able to start, stop and promote Postgres instances in the cluster. Not doing so may result in split-brain scenarios. For example: if the node running as a primary failed and the unit ``postgresql.service`` is enabled, it may bring Postgres back up and cause a split-brain.
+
+Concepts and requirements
+-------------------------
+
+Which are the applications that make part of Patroni?
+    Patroni basically ships a couple applications:
+
+    * ``patroni``: This is the Patroni agent, which takes care of managing a Postgres node;
+    * ``patronictl``: This is a command-line utility used to interact with a Patroni cluster (perform switchovers, restarts, changes in the configuration, etc.). Please find more information in :ref:`patronictl`.
+
+What is a ``standby cluster`` in Patroni?
+    It is a cluster that does not have any primary Postgres node running, i.e., there is no read/write member in the cluster.
+
+    These kinds of clusters exist to replicate data from another cluster and are usually useful when you want to replicate data across data centers.
+
+    There will be a leader in the cluster which will be a standby in charge of replicating changes from a remote Postgres node.
+    Then, there will be a set of standbys configured with cascading replication from such leader member.
+
+    **Note:** the standby cluster doesn't know anything about the source cluster which it is replicating from -- it can even use ``restore_command`` instead of WAL streaming, and may use an absolutely independent DCS cluster.
+
+    Refer to :ref:`standby_cluster` for more details.
+
+What is a ``leader`` in Patroni?
+    A ``leader`` in Patroni is like a coordinator of the cluster.
+
+    In a regular Patroni cluster, the ``leader`` will be the read/write node.
+
+    In a standby Patroni cluster, the ``leader`` (AKA ``standby leader``) will be in charge of replicating from a remote Postgres node, and cascading those changes to the other members of the standby cluster.
+
+Does Patroni require a minimum number of Postgres nodes in the cluster?
+    No, you can run Patroni with any number of Postgres nodes.
+
+    Remember: Patroni is decoupled from the DCS.
+
+What does ``pause`` mean in Patroni?
+    Pause is an operation exposed by Patroni so the user can ask Patroni to step back in regards to Postgres management.
+
+    That is mainly useful when you want to perform maintenance on the cluster, and would like to avoid that Patroni takes decisions related with HA, like failing over to a standby when you stop the primary.
+
+    You can find more information about that in :ref:`pause`.
+
+Automatic failover
+------------------
+
+How does the automatic failover mechanism of Patroni work?
+    Patroni automatic failover is based on what we call ``leader race``.
+
+    Patroni stores the cluster's status in the DCS, among them a ``leader`` lock which holds the name of the Patroni member which is the current ``leader`` of the cluster.
+
+    That ``leader`` lock has a time-to-live associated with it. If the leader node fails to update the lease of the ``leader`` lock in time, the key will eventually expire from the DCS.
+
+    When the ``leader`` lock expires, it triggers what Patroni calls a ``leader race``: all nodes start performing checks to determine if they are the best candidates for taking over the ``leader`` role.
+    Some of these checks include calls to the REST API of all other Patroni members.
+
+    All Patroni members that find themselves as the best candidate for taking over the ``leader`` lock will attempt to do so.
+    The first Patroni member that is able to take the ``leader`` lock will promote itself to a read/write node (or ``standby leader``), and the others will be configured to follow it.
+
+Can I temporarily disable automatic failover in the Patroni cluster?
+    Yes, you can!
+
+    You can achieve that by temporarily pausing the cluster.
+    This is typically useful for performing maintenance.
+
+    When you want to resume the automatic failover of the cluster, you just need to unpause it.
+
+    You can find more information about that in :ref:`pause`.
+
+Bootstrapping and standbys creation
+-----------------------------------
+
+How does Patroni create a primary Postgres node? What about a standby Postgres node?
+    By default Patroni will use ``initdb`` to bootstrap a fresh cluster, and ``pg_basebackup`` to create standby nodes from a copy of the ``leader`` member.
+
+    You can customize that behavior by writing your custom bootstrap methods, and your custom replica creation methods.
+
+    Custom methods are usually useful when you want to restore backups created by backup tools like pgBackRest or Barman, for example.
+
+    For detailed information please refer to :ref:`custom_bootstrap` and :ref:`custom_replica_creation`.
+
+Monitoring
+----------
+
+How can I monitor my Patroni cluster?
+    Patroni exposes a couple handy endpoints in its :ref:`rest_api`:
+
+    * ``/metrics``: exposes monitoring metrics in a format that can be consumed by Prometheus;
+    * ``/patroni``: exposes the status of the cluster in a JSON format. The information shown here is very similar to what is shown by the ``/metrics`` endpoint.
+
+    You can use those endpoints to implement monitoring checks.

docs/ha_multi_dc.rst

@@ -12,7 +12,7 @@ In both cases, it is important to be clear about the following concepts:
 - You should run the odd number of etcd, ZooKeeper or Consul nodes: 3 or 5!
 
 Synchronous Replication
-----------------------------
+-----------------------
 
 To have a multi DC cluster that can automatically tolerate a zone drop, a minimum of 3 is required.
 
@@ -27,7 +27,7 @@ Regarding postgres, we must deploy at least 2 nodes, in different DC. Then you h
 This enables sync replication and the primary node will choose one of the nodes as synchronous.
 
 Asynchronous Replication
-----------------------------------
+------------------------
 
 With only two data centers it would be better to have two independent etcd clusters and run Patroni :ref:`standby cluster <standby_cluster>` in the second data center. If the first site is down, you can MANUALLY promote the ``standby_cluster``.
 

docs/index.rst

@@ -10,7 +10,7 @@ Patroni is a template for high availability (HA) PostgreSQL solutions using Pyth
 
 We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely. There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
 
-Currently supported PostgreSQL versions: 9.3 to 15.
+Currently supported PostgreSQL versions: 9.3 to 16.
 
 **Note to Citus users**: Starting from 3.0 Patroni nicely integrates with the `Citus <https://github.com/citusdata/citus>`__ database extension to Postgres. Please check the :ref:`Citus support page <citus>` in the Patroni documentation for more info about how to use Patroni high availability together with a Citus distributed cluster.
 
@@ -22,24 +22,36 @@ Currently supported PostgreSQL versions: 9.3 to 15.
    :caption: Contents:
 
    README
+   installation
    patroni_configuration
    rest_api
+   patronictl
    replica_bootstrap
    replication_modes
+   standby_cluster
    watchdog
    pause
    dcs_failsafe_mode
    kubernetes
    citus
    existing_data
+   tools_integration
    security
    ha_multi_dc
+   faq
    releases
    CONTRIBUTING
 
 Indices and tables
 ==================
 
-* :ref:`genindex`
+.. ifconfig:: builder == 'html'
-* :ref:`modindex`
+
-* :ref:`search`
+  * :ref:`genindex`
+  * :ref:`modindex`
+  * :ref:`search`
+
+.. ifconfig:: builder != 'html'
+
+  * :ref:`genindex`
+  * :ref:`search`

docs/installation.rst

@@ -0,0 +1,198 @@
+.. _installation:
+
+Installation
+============
+
+Pre-requirements for Mac OS
+---------------------------
+
+To install requirements on a Mac, run the following:
+
+.. code-block:: shell
+
+    brew install postgresql etcd haproxy libyaml python
+
+.. _psycopg2_install_options:
+
+Psycopg
+-------
+
+Starting from `psycopg2-2.8`_ the binary version of psycopg2 will no longer be installed by default. Installing it from
+the source code requires C compiler and postgres+python dev packages. Since in the python world it is not possible to
+specify dependency as ``psycopg2 OR psycopg2-binary`` you will have to decide how to install it.
+
+There are a few options available:
+
+1. Use the package manager from your distro
+
+.. code-block:: shell
+
+    sudo apt-get install python3-psycopg2  # install psycopg2 module on Debian/Ubuntu
+    sudo yum install python3-psycopg2      # install psycopg2 on RedHat/Fedora/CentOS
+
+2. Specify one of `psycopg`, `psycopg2`, or `psycopg2-binary` in the :ref:`list of dependencies <extras>` when installing Patroni with pip.
+
+
+.. _extras:
+
+General installation for pip
+----------------------------
+
+Patroni can be installed with pip:
+
+.. code-block:: shell
+
+    pip install patroni[dependencies]
+
+where ``dependencies`` can be either empty, or consist of one or more of the following:
+
+etcd or etcd3
+    `python-etcd` module in order to use Etcd as Distributed Configuration Store (DCS)
+consul
+    `python-consul` module in order to use Consul as DCS
+zookeeper
+    `kazoo` module in order to use Zookeeper as DCS
+exhibitor
+    `kazoo` module in order to use Exhibitor as DCS (same dependencies as for Zookeeper)
+kubernetes
+    `kubernetes` module in order to use Kubernetes as DCS in Patroni
+raft
+    `pysyncobj` module in order to use python Raft implementation as DCS
+aws
+    `boto3` in order to use AWS callbacks
+jsonlogger
+    `python-json-logger` module in order to enable :ref:`logging <log_settings>` in json format
+all
+    all of the above (except psycopg family)
+psycopg
+    `psycopg[binary]>=3.0.0` module
+psycopg2
+    `psycopg2>=2.5.4` module
+psycopg2-binary
+    `psycopg2-binary` module
+
+For example, the command in order to install Patroni together with psycopg3, dependencies for Etcd as a DCS, and AWS callbacks is:
+
+.. code-block:: shell
+
+    pip install patroni[psycopg3,etcd3,aws]
+
+Note that external tools to call in the replica creation or custom bootstrap scripts (i.e. WAL-E) should be installed
+independently of Patroni.
+
+.. _package_installation:
+
+Package installation on Linux
+-----------------------------
+
+Patroni packages may be available for your operating system, produced by the Postgres community for:
+
+* RHEL, RockyLinux, AlmaLinux;
+* Debian and Ubuntu;
+* SUSE Enterprise Linux.
+
+You can also find packages for direct dependencies of Patroni, like python modules that might not be available in
+the official operating system repositories.
+
+For more information see the `PGDG repository`_ documentation.
+
+If you are on a RedHat Enterprise Linux derivative operating system you may also require packages from EPEL, see
+`EPEL repository`_ documentation.
+
+Once you have installed the PGDG repository for your OS you can install patroni.
+
+.. note::
+
+    Patroni packages are not maintained by the Patroni developers, but rather by the Postgres community. If you
+    require support please first try connecting on `Postgres slack`_.
+
+Installing on Debian derivatives
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+With PGDG repo installed, see :ref:`above <package_installation>`, install Patroni via apt run:
+
+.. code-block:: shell
+
+    apt-get install patroni
+
+Installing on RedHat derivatives
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+With PGDG repo installed, see :ref:`above <package_installation>`, install patroni with an etcd DCS via dnf on RHEL 9
+(and derivatives) run:
+
+.. code-block:: shell
+
+    dnf install patroni patroni-etcd
+
+You can install etcd from PGDG if your RedHat derivative distribution does not provide packages. On the nodes that will
+host the DCS run:
+
+.. code-block:: shell
+
+    dnf install 'dnf-command(config-manager)'
+    dnf config-manager --enable pgdg-rhel9-extras
+    dnf install etcd
+
+You can replace the version of RHEL with `8` in the repo to make `pgdg-rhel8-extras` if needed. The repo name is still
+`pgdg-rhelN-extras` on RockyLinux, AlmaLinux, Oracle Linux, etc...
+
+Installing on SUSE Enterprise Linux
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You might need to enable the SUSE PackageHub repositories for some dependencies. see `SUSE PackageHub`_ documentation.
+
+For SLES 15 with PGDG repo installed, see :ref:`above <package_installation>`, you can install patroni using:
+
+.. code-block:: shell
+
+    zypper install patroni patroni-etcd
+
+With the SUSE PackageHub repo enabled you can also install etcd:
+
+.. code-block:: shell
+
+    SUSEConnect -p PackageHub/15.5/x86_64
+    zypper install etcd
+
+Upgrading
+---------
+
+Upgrading patroni is a very simple process, just update the software installation and restart the Patroni daemon on
+each node in the cluster.
+
+However, restarting the Patroni daemon will result in a Postgres database restart. In some situations this may cause
+a failover of the primary node in your cluster, therefore it is recommended to put the cluster into maintenance mode
+until the Patroni daemon restart has been completed.
+
+To put the cluster in maintenance mode, run the following command on one of the patroni nodes:
+
+.. code-block:: shell
+
+    patronictl pause --wait
+
+Then on each node in the cluster, perform the package upgrade required for your OS:
+
+.. code-block:: shell
+
+    apt-get update && apt-get install patroni patroni-etcd
+
+Restart the patroni daemon process on each node:
+
+.. code-block:: shell
+
+    systemctl restart patroni
+
+Then finally resume monitoring of Postgres with patroni to take it out of maintenance mode:
+
+.. code-block:: shell
+
+    patronictl resume --wait
+
+The cluster will now be full operational with the new version of Patroni.
+
+.. _psycopg2-2.8: http://initd.org/psycopg/articles/2019/04/04/psycopg-28-released/
+.. _PGDG repository: https://www.postgresql.org/download/linux/
+.. _EPEL repository: https://docs.fedoraproject.org/en-US/epel/
+.. _SUSE PackageHub: https://packagehub.suse.com/how-to-use/
+.. _Postgres slack: http://pgtreats.info/slack-invite

docs/kubernetes.rst

@@ -32,6 +32,58 @@ Configuration
 
 Patroni Kubernetes :ref:`settings <kubernetes_settings>` and :ref:`environment variables <kubernetes_environment>` are described in the general chapters of the documentation.
 
+.. _kubernetes_role_values:
+
+Customize role label
+^^^^^^^^^^^^^^^^^^^^
+
+By default, Patroni will set corresponding labels on the pod it runs in based on node's role, such as ``role=master``.
+The key and value of label can be customized by `kubernetes.role_label`, `kubernetes.leader_label_value`, `kubernetes.follower_label_value` and `kubernetes.standby_leader_label_value`.
+
+Note that if you migrate from default role labels to custom ones, you can reduce downtime by following migration steps:
+
+1. Add a temporary label using original role value for the pod with `kubernetes.tmp_role_label` (like ``tmp_role``). Once pods are restarted they will get following labels set by Patroni:
+
+  .. code:: YAML
+
+    labels:
+      cluster-name: foo
+      role: master
+      tmp_role: master
+
+2. After all pods have been updated, modify the service selector to select the temporary label.
+
+  .. code:: YAML
+
+    selector:
+      cluster-name: foo
+      tmp_role: master
+
+3. Add your custom role label (e.g., set `kubernetes.leader_label_value=primary`). Once pods are restarted they will get following new labels set by Patroni:
+
+  .. code:: YAML
+
+    labels:
+      cluster-name: foo
+      role: primary
+      tmp_role: master
+
+4. After all pods have been updated again, modify the service selector to use new role value.
+
+  .. code:: YAML
+
+    selector:
+      cluster-name: foo
+      role: primary
+
+5. Finally, remove the temporary label from your configuration and update all pods.
+
+  .. code:: YAML
+
+    labels:
+      cluster-name: foo
+      role: primary
+
 Examples
 --------
 

docs/patroni_configuration.rst

@@ -15,7 +15,7 @@ There are 3 types of Patroni configuration:
 
 - Global :ref:`dynamic configuration <dynamic_configuration>`.
 	These options are stored in the DCS (Distributed Configuration Store) and applied on all cluster nodes.
-	Dynamic configuration can be set at any time using ``patronictl edit-config`` tool or Patroni :ref:`REST API <rest_api>`.
+	Dynamic configuration can be set at any time using :ref:`patronictl_edit_config` tool or Patroni :ref:`REST API <rest_api>`.
 	If the options changed are not part of the startup configuration, they are applied asynchronously (upon the next wake up cycle)
 	to every node, which gets subsequently reloaded.
 	If the node requires a restart to apply the configuration (for `PostgreSQL parameters <https://www.postgresql.org/docs/current/view-pg-settings.html>`__ with context postmaster, if their values
@@ -24,12 +24,13 @@ There are 3 types of Patroni configuration:
 
 - Local :ref:`configuration file <yaml_configuration>` (patroni.yml).
 	These options are defined in the configuration file and take precedence over dynamic configuration.
-	``patroni.yml`` can be changed and reloaded at runtime (without restart of Patroni) by sending SIGHUP to the Patroni process, performing ``POST /reload`` REST-API request or executing ``patronictl reload``. Local configuration can be either a single YAML file or a directory. When it is a directory, all YAML files in that directory are loaded one by one in sorted order. In case a key is defined in multiple files, the occurrence in the last file takes precedence.
+	``patroni.yml`` can be changed and reloaded at runtime (without restart of Patroni) by sending SIGHUP to the Patroni process, performing ``POST /reload`` REST-API request or executing :ref:`patronictl_reload`. Local configuration can be either a single YAML file or a directory. When it is a directory, all YAML files in that directory are loaded one by one in sorted order. In case a key is defined in multiple files, the occurrence in the last file takes precedence.
 
 - :ref:`Environment configuration <environment>`.
 	It is possible to set/override some of the "Local" configuration parameters with environment variables.
 	Environment configuration is very useful when you are running in a dynamic environment and you don't know some of the parameters in advance (for example it's not possible to know your external IP address when you are running inside ``docker``).
 
+.. _important_configuration_rules:
 
 Important rules
 ---------------
@@ -44,7 +45,6 @@ Some of the PostgreSQL parameters **must hold the same values on the primary and
 - **max_worker_processes**: 8
 - **max_prepared_transactions**: 0
 - **wal_level**: hot_standby
-- **wal_log_hints**: on
 - **track_commit_timestamp**: off
 
 For the parameters below, PostgreSQL does not require equal values among the primary and all the replicas. However, considering the possibility of a replica to become the primary at any time, it doesn't really make sense to set them differently; therefore, **Patroni restricts setting their values to the** :ref:`dynamic configuration <dynamic_configuration>`.
@@ -62,6 +62,7 @@ There are some other Postgres parameters controlled by Patroni:
 - **port** - is set either from ``postgresql.listen`` or from ``PATRONI_POSTGRESQL_LISTEN`` environment variable
 - **cluster_name** - is set either from ``scope`` or from ``PATRONI_SCOPE`` environment variable
 - **hot_standby: on**
+- **wal_log_hints: on** - for Postgres 9.4 and newer.
 
 To be on the safe side parameters from the above lists are not written into ``postgresql.conf``, but passed as a list of arguments to the ``pg_ctl start`` which gives them the highest precedence, even above `ALTER SYSTEM <https://www.postgresql.org/docs/current/static/sql-altersystem.html>`__
 
@@ -70,15 +71,16 @@ There also are some parameters like **postgresql.listen**, **postgresql.data_dir
 
 When applying the local or dynamic configuration options, the following actions are taken:
 
-- The node first checks if there is a `postgresql.base.conf` or if the ``custom_conf`` parameter is set.
+- The node first checks if there is a `postgresql.base.conf` file or if the ``custom_conf`` parameter is set.
-- If the ``custom_conf`` parameter is set, it will take the file specified on it as a base configuration, ignoring `postgresql.base.conf` and `postgresql.conf`.
+- If the ``custom_conf`` parameter is set, the file it specifies is used as the base configuration, ignoring `postgresql.base.conf` and `postgresql.conf`.
-- If the ``custom_conf`` parameter is not set and `postgresql.base.conf` exists, it contains the renamed "original" configuration and it will be used as a base configuration.
+- If the ``custom_conf`` parameter is not set and `postgresql.base.conf` exists, it contains the renamed "original" configuration and is used as the base configuration.
-- If there is no ``custom_conf``` nor `postgresql.base.conf`, the original `postgresql.conf`` is taken and renamed to postgresql.base.conf.
+- If there is no ``custom_conf`` nor `postgresql.base.conf`, the original `postgresql.conf` is renamed to `postgresql.base.conf` and used as the base configuration.
-- The dynamic options (with the exceptions above) are dumped into the `postgresql.conf`` and an include is set in
+- The dynamic options (with the exceptions above) are dumped into the `postgresql.conf` and an include is set in
-  postgresql.conf to the used base configuration (either `postgresql.base.conf` or what is on ``custom_conf``). Therefore, we would be able to apply new options without re-reading the configuration file to check if the include is present not.
+  `postgresql.conf` to the base configuration (either `postgresql.base.conf` or the file at ``custom_conf``).
+  Therefore, we would be able to apply new options without re-reading the configuration file to check if the include is present or not.
 - Some parameters that are essential for Patroni to manage the cluster are overridden using the command line.
-- If some of the options that require restart are changed (we should look at the context in pg_settings and at the actual
+- If an option that requires restart is changed (we should look at the context in pg_settings and at the actual
-  values of those options), a pending_restart flag of a given node is set. This flag is reset on any restart.
+  values of those options), a pending_restart flag is set on that node. This flag is reset on any restart.
 
 The parameters would be applied in the following order (run-time are given the highest priority):
 
@@ -89,6 +91,43 @@ The parameters would be applied in the following order (run-time are given the h
 
 This allows configuration for all the nodes (2), configuration for a specific node using ``ALTER SYSTEM`` (3) and ensures that parameters essential to the running of Patroni are enforced (4), as well as leaves room for configuration tools that manage `postgresql.conf` directly without involving Patroni (1).
 
+.. _shared_memory_gucs:
+
+PostgreSQL parameters that touch shared memory
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+PostgreSQL has some parameters that determine the size of the shared memory used by them:
+
+- **max_connections**
+- **max_prepared_transactions**
+- **max_locks_per_transaction**
+- **max_wal_senders**
+- **max_worker_processes**
+
+Changing these parameters require a PostgreSQL restart to take effect, and their shared memory structures cannot be smaller on the standby nodes than on the primary node.
+
+As explained before, Patroni restrict changing their values through :ref:`dynamic configuration <dynamic_configuration>`, which usually consists of:
+
+1. Applying changes through :ref:`patronictl_edit_config` (or via REST API ``/config`` endpoint)
+2. Restarting nodes through :ref:`patronictl_restart` (or via REST API ``/restart`` endpoint)
+
+**Note:** please keep in mind that you should perform a restart of the PostgreSQL nodes through :ref:`patronictl_restart` command, or via REST API ``/restart`` endpoint. An attempt to restart PostgreSQL by restarting the Patroni daemon, e.g. by executing ``systemctl restart patroni``, can cause a failover to occur in the cluster, if you are restarting the primary node.
+
+However, as those settings manage shared memory, some extra care should be taken when restarting the nodes:
+
+* If you want to **increase** the value of any of those settings:
+
+   1. Restart all standbys first
+   2. Restart the primary after that
+
+* If you want to **decrease** the value of any of those settings:
+
+   1. Restart the primary first
+   2. Restart all standbys after that
+
+**Note:** if you attempt to restart all nodes in one go after **decreasing** the value of any of those settings, Patroni will ignore the change and restart the standby with the original setting value, thus requiring that you restart the standbys again later. Patroni does that to prevent the standby to enter in an infinite crash loop, because PostgreSQL quits with a `FATAL` message if you attempt to set any of those parameters to a value lower than what is visible in ``pg_controldata`` on the Standby node. In other words, we can only decrease the setting on the standby once its ``pg_controldata`` is up-to-date with the primary in regards to these changes on the primary.
+
+More information about that can be found at `PostgreSQL Administrator's Overview <https://www.postgresql.org/docs/current/hot-standby.html#HOT-STANDBY-ADMIN>`__.
 
 Patroni configuration parameters
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -106,3 +145,109 @@ Also the following Patroni configuration options **can be changed only dynamical
 Upon changing these options, Patroni will read the relevant section of the configuration stored in DCS and change its run-time values.
 
 Patroni nodes are dumping the state of the DCS options to disk upon for every change of the configuration into the file ``patroni.dynamic.json`` located in the Postgres data directory. Only the leader is allowed to restore these options from the on-disk dump if these are completely absent from the DCS or if they are invalid.
+
+
+.. _validate_generate_config:
+
+Configuration generation and validation
+---------------------------------------
+
+Patroni provides command-line interfaces for a Patroni :ref:`local configuration <yaml_configuration>` generation and validation. Using the ``patroni`` executable you can:
+
+- Create a sample local Patroni configuration;
+- Create a Patroni configuration file for the locally running PostgreSQL instance (e.g. as a preparation step for the :ref:`Patroni integration <existing_data>`);
+- Validate a given Patroni configuration file.
+
+.. _generate_sample_config:
+
+Sample Patroni configuration
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code:: text
+
+   patroni --generate-sample-config [configfile]
+
+Description
+"""""""""""
+
+Generate a sample Patroni configuration file in ``yaml`` format.
+Parameter values are defined using the :ref:`Environment configuration <environment>`, otherwise, if not set, the defaults used in Patroni or the ``#FIXME`` string for the values that should be later defined by the user.
+
+Some default values are defined based on the local setup:
+
+   -  **postgresql.listen**: the IP address returned by ``gethostname`` call for the current machine's hostname and the standard ``5432`` port.
+   -  **postgresql.connect_address**: the IP address returned by ``gethostname`` call for the current machine's hostname and the standard ``5432`` port.
+   -  **postgresql.authentication.rewind**: is only defined if the PostgreSQL version can be defined from the binary and the version is 11 or later.
+   -  **restapi.listen**: IP address returned by ``gethostname`` call for the current machine's hostname and the standard ``8008`` port.
+   -  **restapi.connect_address**: IP address returned by ``gethostname`` call for the current machine's hostname and the standard ``8008`` port.
+
+Parameters
+""""""""""
+
+``configfile`` - full path to the configuration file used to store the result. If not provided, the result is sent to ``stdout``.
+
+.. _generate_config:
+
+Patroni configuration for a running instance
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code:: text
+
+   patroni --generate-config [--dsn DSN] [configfile]
+
+Description
+"""""""""""
+
+Generate a Patroni configuration in ``yaml`` format for the locally running PostgreSQL instance. 
+Either the provided DSN (takes precedence) or PostgreSQL `environment variables <https://www.postgresql.org/docs/current/libpq-envars.html>`__ will be used for the PostgreSQL connection. If the password is not provided, it should be entered via prompt.
+
+All the non-internal GUCs defined in the source Postgres instance, independently if they were set through a configuration file, through the postmaster command-line, or through environment variables, will be used as the source for the following Patroni configuration parameters:
+
+   -  **scope**: ``cluster_name`` GUC value;
+   -  **postgresql.listen**: ``listen_addresses`` and ``port`` GUC values;
+   -  **postgresql.datadir**: ``data_directory`` GUC value;
+   -  **postgresql.parameters**: ``archive_command``, ``restore_command``, ``archive_cleanup_command``, ``recovery_end_command``, ``ssl_passphrase_command``, ``hba_file``, ``ident_file``, ``config_file`` GUC values;
+   -  **bootstrap.dcs**: all other gathered PostgreSQL GUCs.
+
+If ``scope``, ``postgresql.listen`` or ``postgresql.datadir`` is not set from the Postgres GUCs, the respective :ref:`Environment configuration <environment>` value is used.
+
+Other rules applied for the values definition:
+
+   -  **name**: ``PATRONI_NAME`` environment variable value if set, otherwise the current machine's hostname.
+   -  **postgresql.bin_dir**: path to the Postgres binaries gathered from the running instance.
+   -  **postgresql.connect_address**: the IP address returned by ``gethostname`` call for the current machine's hostname and the port used for the instance connection or the ``port`` GUC value.
+   -  **postgresql.authentication.superuser**: the configuration used for the instance connection;
+   -  **postgresql.pg_hba**: the lines gathered from the source instance's ``hba_file``.
+   -  **postgresql.pg_ident**: the lines gathered from the source instance's ``ident_file``.
+   -  **restapi.listen**: IP address returned by ``gethostname`` call for the current machine's hostname and the standard ``8008`` port.
+   -  **restapi.connect_address**: IP address returned by ``gethostname`` call for the current machine's hostname and the standard ``8008`` port.
+
+Other parameters defined using :ref:`Environment configuration <environment>` are also included into the configuration.
+
+Parameters
+""""""""""
+
+``configfile``
+    Full path to the configuration file used to store the result. If not provided, result is sent to ``stdout``.
+
+``dsn``
+    Optional DSN string for the local PostgreSQL instance to get GUC values from.
+
+
+Validate Patroni configuration
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code:: text
+
+   patroni --validate-config [configfile]
+
+Description
+"""""""""""
+
+Validate the given Patroni configuration and print the information about the failed checks.
+
+Parameters
+""""""""""
+
+``configfile``
+    Full path to the configuration file to check. If not given or file does not exist, will try to read from the ``PATRONI_CONFIG_VARIABLE`` environment variable or, if not set, from the :ref:`Patroni environment variables <environment>`.

docs/pause.rst

@@ -19,7 +19,7 @@ When Patroni runs in a paused mode, it does not change the state of PostgreSQL,
 
 - For the Postgres primary with the leader lock Patroni updates the lock. If the node with the leader lock stops being the primary (i.e. is demoted manually), Patroni will release the lock instead of promoting the node back.
 
-- Manual unscheduled restart, reinitialize and manual failover are allowed. Manual failover is only allowed if the node to failover to is specified. In the paused mode, manual failover does not require a running primary node.
+- Manual unscheduled restart, manual unscheduled failover/switchover and reinitialize are allowed. No scheduled action is allowed. Manual switchover is only allowed if the node to switch over to is specified.
 
 - If 'parallel' primaries are detected by Patroni, it emits a warning, but does not demote the primary without the leader lock.
 
@@ -32,6 +32,6 @@ When Patroni runs in a paused mode, it does not change the state of PostgreSQL,
 User guide
 ----------
 
-``patronictl`` supports ``pause`` and ``resume`` commands.
+``patronictl`` supports :ref:`pause <patronictl_pause>` and :ref:`resume <patronictl_resume>` commands.
 
 One can also issue a ``PATCH`` request to the ``{namespace}/{cluster}/config`` key with ``{"pause": true/false/null}``

docs/releases.rst

@@ -3,6 +3,324 @@
 Release notes
 =============
 
+Version 3.2.2
+-------------
+
+**Bugfixes**
+
+- Don't let replica restore initialize key when DCS was wiped (Alexander Kukushkin)
+
+  It was happening in the method where Patroni was supposed to take over a standalone PG cluster.
+
+- Use consistent read when fetching just updated sync key from Consul (Alexander Kukushkin)
+
+  Consul doesn't provide any interface to immediately get ``ModifyIndex`` for the key that we just updated, therefore we have to perform an explicit read operation. Since stale reads are allowed by default, we sometimes used to get an outdated version of the key.
+
+- Reload Postgres config if a parameter that requires restart was reset to the original value (Polina Bungina)
+
+  Previously Patroni wasn't updating the config, but only resetting the ``pending_restart``.
+
+- Fix erroneous inverted logic of the confirmation prompt message when doing a failover to an async candidate in synchronous mode (Polina Bungina)
+
+  The problem existed only in ``patronictl``.
+
+- Exclude leader from failover candidates in ``patronictl`` (Polina Bungina)
+
+  If the cluster is healthy, failing over to an existing leader is no-op.
+
+- Create Citus database and extension idempotently (Alexander Kukushkin, Zhao Junwang)
+
+  It will allow to create them in the ``post_bootstrap`` script in case if there is a need to add some more dependencies to the Citus database.
+
+- Don't filter our contradictory ``nofailover`` tag (Polina Bungina)
+
+  The configuration ``{nofailover: false, failover_priority: 0}`` set on a node didn't allow it to participate in the race, while it should, because ``nofailover`` tag should take precedence.
+
+- Fixed PyInstaller frozen issue (Sophia Ruan)
+
+  The ``freeze_support()`` was called after ``argparse`` and as a result, Patroni wasn't able to start Postgres.
+
+- Fixed bug in the config generator for ``patronictl`` and ``Citus`` configuration (Israel Barth Rubio)
+
+  It prevented ``patronictl`` and ``Citus`` configuration parameters set via environment variables from being written into the generated config.
+
+- Restore recovery GUCs and some Patroni-managed parameters when joining a running standby (Alexander Kukushkin)
+
+  Patroni was failing to restart Postgres v12 onwards with an error about missing ``port`` in one of the internal structures.
+
+- Fixes around ``pending_restart`` flag (Polina Bungina)
+
+  Don't expose ``pending_restart`` when in custom bootstrap with ``recovery_target_action = promote`` or when someone changed ``hot_standby`` or ``wal_log_hints`` using for example ``ALTER SYSTEM``.
+
+
+Version 3.2.1
+-------------
+
+**Bugfixes**
+
+- Limit accepted values for ``--format`` argument in ``patronictl`` (Alexander Kukushkin)
+
+  It used to accept any arbitrary string and produce no output if the value wasn't recognized.
+
+- Verify that replica nodes received checkpoint LSN on shutdown before releasing the leader key (Alexander Kukushkin)
+
+  Previously in some cases, we were using LSN of the SWITCH record that is followed by CHECKPOINT (if archiving mode is enabled). As a result the former primary sometimes had to do ``pg_rewind``, but there would be no data loss involved.
+
+- Do a real HTTP request when performing node name uniqueness check (Alexander Kukushkin)
+
+  When running Patroni in containers it is possible that the traffic is routed using ``docker-proxy``, which listens on the port and accepts incoming connections. It was causing false positives.
+
+- Fixed Citus support with Etcd v2 (Alexander Kukushkin)
+
+  Patroni was failing to deploy a new Citus cluster with Etcd v2.
+
+- Fixed ``pg_rewind`` behavior with Postgres v16+ (Alexander Kukushkin)
+
+  The error message format of ``pg_waldump`` changed in v16 which caused ``pg_rewind`` to be called by Patroni even when it was not necessary.
+
+- Fixed bug with custom bootstrap (Alexander Kukushkin)
+
+  Patroni was falsely applying ``--command`` argument, which is a bootstrap command itself.
+
+- Fixed the issue with REST API health check endpoints (Sophia Ruan)
+
+  There were chances that after Postgres restart it could return ``unknown`` state for Postgres because connections were not properly closed.
+
+- Cache ``postgres --describe-config`` output results (Waynerv)
+
+  They are used to figure out which GUCs are available to validate PostgreSQL configuration and we don't expect this list to change while Patroni is running.
+
+
+Version 3.2.0
+-------------
+
+**Deprecation notice**
+
+- The ``bootstrap.users`` support will be removed in version 4.0.0. If you need to create users after deploying a new cluster please use the ``bootstrap.post_bootstrap`` hook for that.
+
+
+**Breaking changes**
+
+- Enforce ``loop_wait + 2*retry_timeout <= ttl`` rule and hard-code minimal possible values (Alexander Kukushkin)
+
+  Minimal values: ``loop_wait=2``, ``retry_timeout=3``, ``ttl=20``. In case values are smaller or violate the rule they are adjusted and a warning is written to Patroni logs.
+
+
+**New features**
+
+- Failover priority (Mark Pekala)
+
+  With the help of ``tags.failover_priority`` it's now possible to make a node more preferred during the leader race. More details in the documentation (ref tags).
+
+- Implemented ``patroni --generate-config [--dsn DSN]`` and ``patroni --generate-sample-config`` (Polina Bungina)
+
+  It allows to generate a config file for the running PostgreSQL cluster or a sample config file for the new Patroni cluster.
+
+- Use a dedicated connection to Postgres for Patroni REST API (Alexander Kukushkin)
+
+  It helps to avoid blocking the main heartbeat loop if the system is under stress.
+
+- Enrich some endpoints with the ``name`` of the node (sskserk)
+
+  For the monitoring endpoint ``name`` is added next to the ``scope`` and for metrics endpoint the ``name`` is added to tags.
+
+- Ensure strict failover/switchover difference (Polina Bungina)
+
+  Be more precise in log messages and allow failing over to an asynchronous node in a healthy synchronous cluster.
+
+- Make permanent physical replication slots behave similarly to permanent logical slots (Alexander Kukushkin)
+
+  Create permanent physical replication slots on all nodes that are allowed to become the leader and use ``pg_replication_slot_advance()`` function to advance ``restart_lsn`` for slots on standby nodes.
+
+- Add capability of specifying namespace through ``--dcs`` argument in ``patronictl`` (Israel Barth Rubio)
+
+  It could be handy if ``patronictl`` is used without a configuration file.
+
+- Add support for additional parameters in custom bootstrap configuration (Israel Barth Rubio)
+
+  Previously it was only possible to add custom arguments to the ``command`` and now one could list them as a mapping.
+
+
+**Improvements**
+
+- Set ``citus.local_hostname`` GUC to the same value which is used by Patroni to connect to the Postgres (Alexander Kukushkin)
+
+  There are cases when Citus wants to have a connection to the local Postgres. By default it uses ``localhost``, which is not always available.
+
+
+**Bugfixes**
+
+- Ignore ``synchronous_mode`` setting in a standby cluster (Polina Bungina)
+
+  Postgres doesn't support cascading synchronous replication and not ignoring ``synchronous_mode`` was breaking a switchover in a standby cluster.
+
+- Handle SIGCHLD for ``on_reload`` callback (Alexander Kukushkin)
+
+  Not doing so results in a zombie process, which is reaped only when the next ``on_reload`` is executed.
+
+- Handle ``AuthOldRevision`` error when working with Etcd v3 (Alexander Kukushkin, Kenny Do)
+
+  The error is raised if Etcd is configured to use JWT and when the user database in Etcd is updated.
+
+
+Version 3.1.2
+-------------
+
+**Bugfixes**
+
+- Fixed bug with ``wal_keep_size`` checks (Alexander Kukushkin)
+
+  The ``wal_keep_size`` is a GUC that normally has a unit and Patroni was failing to cast its value to ``int``. As a result the value of ``bootstrap.dcs`` was not written to the ``/config`` key afterwards.
+
+- Detect and resolve inconsistencies between ``/sync`` key and ``synchronous_standby_names`` (Alexander Kukushkin)
+
+  Normally, Patroni updates ``/sync`` and ``synchronous_standby_names`` in a very specific order, but in case of a bug or when someone manually reset ``synchronous_standby_names``, Patroni was getting into an inconsistent state. As a result it was possible that the failover happens to an asynchronous node.
+
+- Read GUC's values when joining running Postgres (Alexander Kukushkin)
+
+  When restarted in ``pause``, Patroni was discarding the ``synchronous_standby_names`` GUC from the ``postgresql.conf``. To solve it and avoid similar issues, Patroni will read GUC's value if it is joining an already running Postgres.
+
+- Silenced annoying warnings when checking for node uniqueness (Alexander Kukushkin)
+
+  ``WARNING`` messages are produced by ``urllib3`` if Patroni is quickly restarted.
+
+
+Version 3.1.1
+-------------
+
+**Bugfixes**
+
+- Reset failsafe state on promote (ChenChangAo)
+
+  If switchover/failover happened shortly after failsafe mode had been activated, the newly promoted primary was demoting itself after failsafe becomes inactive.
+
+- Silence useless warnings in ``patronictl`` (Alexander Kukushkin)
+
+  If ``patronictl`` uses the same patroni.yaml file as Patroni and can access ``PGDATA`` directory it might have been showing annoying warnings about incorrect values in the global configuration.
+
+- Explicitly enable synchronous mode for a corner case (Alexander Kukushkin)
+
+  Synchronous mode effectively was never activated if there are no replicas streaming from the primary.
+
+- Fixed bug with ``0`` integer values validation (Israel Barth Rubio)
+
+  In most cases, it didn't cause any issues, just warnings.
+
+- Don't return logical slots for standby cluster (Alexander Kukushkin)
+
+  Patroni can't create logical replication slots in the standby cluster, thus they should be ignored if they are defined in the global configuration.
+
+- Avoid showing docstring in ``patronictl --help`` output (Israel Barth Rubio)
+
+  The ``click`` module needs to get a special hint for that.
+
+- Fixed bug with ``kubernetes.standby_leader_label_value`` (Alexander Kukushkin)
+
+  This feature effectively never worked.
+
+- Returned cluster system identifier to the ``patronictl list`` output (Polina Bungina)
+
+  The problem was introduced while implementing the support for Citus, where we need to hide the identifier because it is different for coordinator and all workers.
+
+- Override ``write_leader_optime`` method in Kubernetes implementation (Alexander Kukushkin)
+
+  The method is supposed to write shutdown LSN to the leader Endpoint/ConfigMap when there are no healthy replicas available to become the new primary.
+
+- Don't start stopped postgres in pause (Alexander Kukushkin)
+
+  Due to a race condition, Patroni was falsely assuming that the standby should be restarted because some recovery parameters (``primary_conninfo`` or similar) were changed.
+
+- Fixed bug in ``patronictl query`` command (Israel Barth Rubio)
+
+  It didn't work when only ``-m`` argument was provided or when none of ``-r`` or ``-m`` were provided.
+
+- Properly treat integer parameters that are used in the command line to start postgres (Polina Bungina)
+
+  If values are supplied as strings and not casted to integer it was resulting in an incorrect calculation of ``max_prepared_transactions`` based on ``max_connections`` for Citus clusters.
+
+- Don't rely on ``pg_stat_wal_receiver`` when deciding on ``pg_rewind`` (Alexander Kukushkin)
+
+  It could happen that ``received_tli`` reported by ``pg_stat_wal_recevier`` is ahead of the actual replayed timeline, while the timeline reported by ``DENTIFY_SYSTEM`` via replication connection is always correct.
+
+
+Version 3.1.0
+-------------
+
+**Breaking changes**
+
+- Changed semantic of ``restapi.keyfile`` and ``restapi.certfile`` (Alexander Kukushkin)
+
+  Previously Patroni was using ``restapi.keyfile`` and ``restapi.certfile`` as client certificates as a fallback if there were no respective configuration parameters in the ``ctl`` section.
+
+.. warning::
+    If you enabled client certificates validation (``restapi.verify_client`` is set to ``required``), you also **must** provide **valid client certificates** in the ``ctl.certfile``, ``ctl.keyfile``, ``ctl.keyfile_password``. If not provided, Patroni will not work correctly.
+
+
+**New features**
+
+- Make Pod role label configurable (Waynerv)
+
+  Values could be customized using ``kubernetes.leader_label_value``, ``kubernetes.follower_label_value`` and ``kubernetes.standby_leader_label_value`` parameters. This feature will be very useful when we change the ``master`` role to the ``primary``. You can read more about the feature and migration steps :ref:`here <kubernetes_role_values>`.
+
+
+**Improvements**
+
+- Various improvements of ``patroni --validate-config`` (Alexander Kukushkin)
+
+  Improved parameter validation for different DCS, ``bootstrap.dcs`` , ``ctl``, ``restapi``, and ``watchdog`` sections.
+
+- Start Postgres not in recovery if it crashed during recovery while Patroni is running (Alexander Kukushkin)
+
+  It may reduce recovery time and will help to prevent unnecessary timeline increments.
+
+- Avoid unnecessary updates of ``/status`` key (Alexander Kukushkin)
+
+  When there are no permanent logical slots Patroni was updating the ``/status`` on every heartbeat loop even when LSN on the primary didn't move forward.
+
+- Don't allow stale primary to win the leader race (Alexander Kukushkin)
+
+  If Patroni was hanging during a significant time due to lack of resources it will additionally check that no other nodes promoted Postgres before acquiring the leader lock.
+
+- Implemented visibility of certain PostgreSQL parameters validation (Alexander Kukushkin, Feike Steenbergen)
+
+  If validation of ``max_connections``, ``max_wal_senders``, ``max_prepared_transactions``, ``max_locks_per_transaction``, ``max_replication_slots``, or ``max_worker_processes`` failed Patroni was using some sane default value. Now in addition to that it will also show a warning.
+
+- Set permissions for files and directories created in ``PGDATA`` (Alexander Kukushkin)
+
+  All files created by Patroni had only owner read/write permissions. This behaviour was breaking backup tools that run under a different user and relying on group read permissions. Now Patroni honors permissions on ``PGDATA`` and correctly sets permissions on all directories and files it creates inside ``PGDATA``.
+
+
+**Bugfixes**
+
+- Run ``archive_command`` through shell (Waynerv)
+
+  Patroni might archive some WAL segments before doing crash recovery in a single-user mode or before ``pg_rewind``. If the archive_command contains some shell operators, like ``&&`` it didn't work with Patroni.
+
+- Fixed "on switchover" shutdown checks (Polina Bungina)
+
+  It was possible that specified candidate is still streaming and didn't received shut down checking but the leader key was removed because some other nodes were healthy.
+
+- Fixed "is primary" check (Alexander Kukushkin)
+
+  During the leader race replicas were not able to recognize that Postgres on the old leader is still running as a primary.
+
+- Fixed ``patronictl list`` (Alexander Kukushkin)
+
+  The Cluster name field was missing in ``tsv``, ``json``, and ``yaml`` output formats.
+
+- Fixed ``pg_rewind`` behaviour after pause (Alexander Kukushkin)
+
+  Under certain conditions, Patroni wasn't able to join the false primary back to the cluster with ``pg_rewind`` after coming out of maintenance mode.
+
+- Fixed bug in Etcd v3 implementation (Alexander Kukushkin)
+
+  Invalidate internal KV cache if key update performed using ``create_revision``/``mod_revision`` field due to revision mismatch.
+
+- Fixed behaviour of replicas in standby cluster in pause (Alexander Kukushkin)
+
+  When the leader key expires replicas in standby cluster will not follow the remote node but keep ``primary_conninfo`` as it is.
+
+
 Version 3.0.4
 -------------
 

docs/replica_bootstrap.rst

@@ -1,3 +1,5 @@
+.. _replica_imaging_and_bootstrap:
+
 Replica imaging and bootstrap
 =============================
 
@@ -43,19 +45,49 @@ in the configuration files, Patroni supplies two cluster-specific ones:
 
 Passing these two additional flags can be disabled by setting a special ``no_params`` parameter to ``True``.
 
-If the bootstrap script returns 0, Patroni tries to configure and start the PostgreSQL instance produced by it. If any
+If the bootstrap script returns ``0``, Patroni tries to configure and start the PostgreSQL instance produced by it. If any
 of the intermediate steps fail, or the script returns a non-zero value, Patroni assumes that the bootstrap has failed,
 cleans up after itself and releases the initialize lock to give another node the opportunity to bootstrap.
 
 If a ``recovery_conf`` block is defined in the same section as the custom bootstrap method, Patroni will generate a
-``recovery.conf`` before starting the newly bootstrapped instance. Typically, such recovery.conf should contain at least
+``recovery.conf`` before starting the newly bootstrapped instance (or set the recovery settings on Postgres configuration if
-one of the ``recovery_target_*`` parameters, together with the ``recovery_target_timeline`` set to ``promote``.
+running PostgreSQL >= 12).
+Typically, such recovery configuration should contain at least one of the ``recovery_target_*`` parameters, together with the ``recovery_target_timeline`` set to ``promote``.
 
-If ``keep_existing_recovery_conf`` is defined and set to ``True``, Patroni will not remove the existing ``recovery.conf`` file if it exists.
+If ``keep_existing_recovery_conf`` is defined and set to ``True``, Patroni will not remove the existing ``recovery.conf`` file if it exists (PostgreSQL <= 11).
-This is useful when bootstrapping from a backup with tools like pgBackRest that generate the appropriate ``recovery.conf`` for you.
+Similarly, in that case Patroni will not remove the existing ``recovery.signal`` or ``standby.signal`` if either exists, nor will it override the configured recovery settings (PostgreSQL >= 12).
+This is useful when bootstrapping from a backup with tools like pgBackRest that generate the appropriate recovery configuration for you.
+
+Besides that, any additional key/value pairs informed in the custom bootstrap method configuration will be passed as arguments to ``command`` in the format ``--name=value``. For example:
+
+.. code:: YAML
+
+    bootstrap:
+        method: <custom_bootstrap_method_name>
+        <custom_bootstrap_method_name>:
+            command: <path_to_custom_bootstrap_script>
+            arg1: value1
+            arg2: value2
+
+Makes the configured ``command`` to be called additionally with ``--arg1=value1 --arg2=value2`` command-line arguments.
 
  .. note:: Bootstrap methods are neither chained, nor fallen-back to the default one in case the primary one fails
 
+As an example, you are able to bootstrap a fresh Patroni cluster from a Barman backup with a configuration like this:
+
+.. code:: YAML
+
+    bootstrap:
+        method: barman
+        barman:
+            keep_existing_recovery_conf: true
+            command: patroni_barman --api-url https://barman-host:7480 recover
+            barman-server: my_server
+            ssh-command: ssh postgres@patroni-host
+
+.. note::
+    ``patroni_barman recover`` requires that you have both Barman and ``pg-backup-api`` configured in the Barman host, so it can execute a remote ``barman recover`` through the backup API.
+    The above example uses a subset of the available parameters. You can get more information running ``patroni_barman recover --help``.
 
 .. _custom_replica_creation:
 
@@ -110,6 +142,24 @@ example: pgbackrest
         basebackup:
             max-rate: '100M'
 
+example: Barman
+
+.. code:: YAML
+
+    postgresql:
+        create_replica_methods:
+            - barman
+            - basebackup
+        barman:
+            command: patroni_barman --api-url https://barman-host:7480 recover
+            barman-server: my_server
+            ssh-command: ssh postgres@patroni-host
+        basebackup:
+            max-rate: '100M'
+
+.. note::
+    ``patroni_barman recover`` requires that you have both Barman and ``pg-backup-api`` configured in the Barman host, so it can execute a remote ``barman recover`` through the backup API.
+    The above example uses a subset of the available parameters. You can get more information running ``patroni_barman recover --help``.
 
 The ``create_replica_methods`` defines available replica creation methods and the order of executing them. Patroni will
 stop on the first one that returns 0. Each method should define a separate section in the configuration file, listing the command
@@ -169,59 +219,3 @@ and
             - waldir: /pg-wal-mount/external-waldir
 
 If all replica creation methods fail, Patroni will try again all methods in order during the next event loop cycle.
-
-.. _standby_cluster:
-
-Standby cluster
----------------
-
-Another available option is to run a "standby cluster", that contains only of
-standby nodes replicating from some remote node. This type of clusters has:
-
-* "standby leader", that behaves pretty much like a regular cluster leader,
-  except it replicates from a remote node.
-
-* cascade replicas, that are replicating from standby leader.
-
-Standby leader holds and updates a leader lock in DCS. If the leader lock
-expires, cascade replicas will perform an election to choose another leader
-from the standbys.
-
-There is no further relationship between the standby cluster and the primary
-cluster it replicates from, in particular, they must not share the same DCS
-scope if they use the same DCS. They do not know anything else from each other
-apart from replication information. Also, the standby cluster is not being
-displayed in ``patronictl list`` or ``patronictl topology`` output on the
-primary cluster.
-
-For the sake of flexibility, you can specify methods of creating a replica and
-recovery WAL records when a cluster is in the "standby mode" by providing
-`create_replica_methods` key in `standby_cluster` section. It is distinct from
-creating replicas, when cluster is detached and functions as a normal cluster,
-which is controlled by `create_replica_methods` in `postgresql` section. Both
-"standby" and "normal" `create_replica_methods` reference  keys in `postgresql`
-section.
-
-To configure such cluster you need to specify the section ``standby_cluster``
-in a patroni configuration:
-
-.. code:: YAML
-
-    bootstrap:
-        dcs:
-            standby_cluster:
-                host: 1.2.3.4
-                port: 5432
-                primary_slot_name: patroni
-                create_replica_methods:
-                - basebackup
-
-Note, that these options will be applied only once during cluster bootstrap,
-and the only way to change them afterwards is through DCS.
-
-Patroni expects to find `postgresql.conf` or `postgresql.conf.backup` in PGDATA
-of the remote primary and will not start if it does not find it after a
-basebackup. If the remote primary keeps its `postgresql.conf` elsewhere, it is
-your responsibility to copy it to PGDATA.
-
-If you use replication slots on the standby cluster, you must also create the corresponding replication slot on the primary cluster.  It will not be done automatically by the standby cluster implementation.  You can use Patroni's permanent replication slots feature on the primary cluster to maintain a replication slot with the same name as ``primary_slot_name``, or its default value if ``primary_slot_name`` is not provided.

docs/replication_modes.rst

@@ -6,8 +6,9 @@ Replication modes
 
 Patroni uses PostgreSQL streaming replication. For more information about streaming replication, see the `Postgres documentation <http://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION>`__. By default Patroni configures PostgreSQL for asynchronous replication. Choosing your replication schema is dependent on your business considerations. Investigate both async and sync replication, as well as other HA solutions, to determine which solution is best for you.
 
+
 Asynchronous mode durability
-----------------------------
+============================
 
 In asynchronous mode the cluster is allowed to lose some committed transactions to ensure availability. When the primary server fails or becomes unavailable for any other reason Patroni will automatically promote a sufficiently healthy standby to primary. Any transactions that have not been replicated to that standby remain in a "forked timeline" on the primary, and are effectively unrecoverable [1]_.
 
@@ -15,10 +16,11 @@ The amount of transactions that can be lost is controlled via ``maximum_lag_on_f
 
 By default, when running leader elections, Patroni does not take into account the current timeline of replicas, what in some cases could be undesirable behavior. You can prevent the node not having the same timeline as a former primary become the new leader by changing the value of ``check_timeline`` parameter to ``true``.
 
-PostgreSQL synchronous replication
-----------------------------------
 
-You can use Postgres's `synchronous replication <http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION>`__ with Patroni. Synchronous replication ensures consistency across a cluster by confirming that writes are written to a secondary before returning to the connecting client with a success. The cost of synchronous replication: reduced throughput on writes. This throughput will be entirely based on network performance.
+PostgreSQL synchronous replication
+==================================
+
+You can use Postgres's `synchronous replication <http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION>`__ with Patroni. Synchronous replication ensures consistency across a cluster by confirming that writes are written to a secondary before returning to the connecting client with a success. The cost of synchronous replication: increased latency and reduced throughput on writes. This throughput will be entirely based on network performance.
 
 In hosted datacenter environments (like AWS, Rackspace, or any network you do not control), synchronous replication significantly increases the variability of write performance. If followers become inaccessible from the leader, the leader effectively becomes read-only.
 
@@ -33,10 +35,11 @@ When using PostgreSQL synchronous replication, use at least three Postgres data
 
 Using PostgreSQL synchronous replication does not guarantee zero lost transactions under all circumstances. When the primary and the secondary that is currently acting as a synchronous replica fail simultaneously a third node that might not contain all transactions will be promoted.
 
+
 .. _synchronous_mode:
 
 Synchronous mode
-----------------
+================
 
 For use cases where losing committed transactions is not permissible you can turn on Patroni's ``synchronous_mode``. When ``synchronous_mode`` is turned on Patroni will not promote a standby unless it is certain that the standby contains all transactions that may have returned a successful commit status to client [2]_. This means that the system may be unavailable for writes even though some servers are available. System administrators can still use manual failover commands to promote a standby even if it results in transaction loss.
 
@@ -53,30 +56,124 @@ are available. As a downside, the primary is not be available for writes
 blocking all client write requests until at least one synchronous replica comes
 up.
 
-You can ensure that a standby never becomes the synchronous standby by setting ``nosync`` tag to true. This is recommended to set for standbys that are behind slow network connections and would cause performance degradation when becoming a synchronous standby.
+You can ensure that a standby never becomes the synchronous standby by setting ``nosync`` tag to true. This is recommended to set for standbys that are behind slow network connections and would cause performance degradation when becoming a synchronous standby. Setting tag ``nostream`` to true will also have the same effect.
 
-Synchronous mode can be switched on and off via Patroni REST interface. See :ref:`dynamic configuration <dynamic_configuration>` for instructions.
+Synchronous mode can be switched on and off using ``patronictl edit-config`` command or via Patroni REST interface. See :ref:`dynamic configuration <dynamic_configuration>` for instructions.
 
 Note: Because of the way synchronous replication is implemented in PostgreSQL it is still possible to lose transactions even when using ``synchronous_mode_strict``. If the PostgreSQL backend is cancelled while waiting to acknowledge replication (as a result of packet cancellation due to client timeout or backend failure) transaction changes become visible for other backends. Such changes are not yet replicated and may be lost in case of standby promotion.
 
+
 Synchronous Replication Factor
-------------------------------
+==============================
-The parameter ``synchronous_node_count`` is used by Patroni to manage number of synchronous standby databases. It is set to 1 by default. It has no effect when ``synchronous_mode`` is set to off. When enabled, Patroni manages precise number of synchronous standby databases based on parameter ``synchronous_node_count`` and adjusts the state in DCS & synchronous_standby_names as members join and leave.
+
+The parameter ``synchronous_node_count`` is used by Patroni to manage the number of synchronous standby databases. It is set to ``1`` by default. It has no effect when ``synchronous_mode`` is set to ``off``. When enabled, Patroni manages the precise number of synchronous standby databases based on parameter ``synchronous_node_count`` and adjusts the state in DCS & ``synchronous_standby_names`` in PostgreSQL as members join and leave. If the parameter is set to a value higher than the number of eligible nodes it will be automatically reduced by Patroni.
+
+
+Maximum lag on synchronous node
+===============================
+
+By default Patroni sticks to nodes that are declared as ``synchronous``, according to the ``pg_stat_replication`` view, even when there are other nodes ahead of it. This is done to minimize the number of changes of ``synchronous_standby_names``. To change this behavior one may use ``maximum_lag_on_syncnode`` parameter. It controls how much lag the replica can have to still be considered as "synchronous".
+
+Patroni utilizes the max replica LSN if there is more than one standby, otherwise it will use leader's current wal LSN. The default is ``-1``, and Patroni will not take action to swap a synchronous unhealthy standby when the value is set to ``0`` or less. Please set the value high enough so that Patroni won't swap synchronous standbys frequently during high transaction volume.
+
 
 Synchronous mode implementation
--------------------------------
+===============================
 
-When in synchronous mode Patroni maintains synchronization state in the DCS, containing the latest primary and current synchronous standby databases. This state is updated with strict ordering constraints to ensure the following invariants:
+When in synchronous mode Patroni maintains synchronization state in the DCS (``/sync`` key), containing the latest primary and current synchronous standby databases. This state is updated with strict ordering constraints to ensure the following invariants:
 
 - A node must be marked as the latest leader whenever it can accept write transactions. Patroni crashing or PostgreSQL not shutting down can cause violations of this invariant.
 
-- A node must be set as the synchronous standby in PostgreSQL as long as it is published as the synchronous standby.
+- A node must be set as the synchronous standby in PostgreSQL as long as it is published as the synchronous standby in the ``/sync`` key in DCS..
 
 - A node that is not the leader or current synchronous standby is not allowed to promote itself automatically.
 
 Patroni will only assign one or more synchronous standby nodes based on ``synchronous_node_count`` parameter to ``synchronous_standby_names``.
 
-On each HA loop iteration Patroni re-evaluates synchronous standby nodes choice. If the current list of synchronous standby nodes are connected and has not requested its synchronous status to be removed it remains picked. Otherwise the cluster member available for sync that is furthest ahead in replication is picked.
+On each HA loop iteration Patroni re-evaluates synchronous standby nodes choice. If the current list of synchronous standby nodes are connected and has not requested its synchronous status to be removed it remains picked. Otherwise the cluster members available for sync that are furthest ahead in replication are picked.
+
+Example:
+---------
+
+``/config`` key in DCS
+^^^^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: YAML
+
+    synchronous_mode: on
+    synchronous_node_count: 2
+    ...
+
+``/sync`` key in DCS
+^^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: JSON
+
+    {
+        "leader": "node0",
+        "sync_standby": "node1,node2"
+    }
+
+postgresql.conf
+^^^^^^^^^^^^^^^
+
+.. code-block:: INI
+
+    synchronous_standby_names = 'FIRST 2 (node1,node2)'
+
+
+In the above examples only nodes ``node1`` and ``node2`` are known to be synchronous and allowed to be automatically promoted if the primary (``node0``) fails.
+
+
+.. _quorum_mode:
+
+Quorum commit mode
+==================
+
+Starting from PostgreSQL v10 Patroni supports quorum-based synchronous replication.
+
+In this mode, Patroni maintains synchronization state in the DCS, containing the latest known primary, the number of nodes required for quorum, and the nodes currently eligible to vote on quorum. In steady state, the nodes voting on quorum are the leader and all synchronous standbys. This state is updated with strict ordering constraints, with regards to node promotion and ``synchronous_standby_names``, to ensure that at all times any subset of voters that can achieve quorum includes at least one node with the latest successful commit.
+
+On each iteration of HA loop, Patroni re-evaluates synchronous standby choices and quorum, based on node availability and requested cluster configuration. In PostgreSQL versions above 9.6 all eligible nodes are added as synchronous standbys as soon as their replication catches up to leader.
+
+Quorum commit helps to reduce worst case latencies, even during normal operation, as a higher latency of replicating to one standby can be compensated by other standbys.
+
+The quorum-based synchronous mode could be enabled by setting ``synchronous_mode`` to ``quorum`` using ``patronictl edit-config`` command or via Patroni REST interface. See :ref:`dynamic configuration <dynamic_configuration>` for instructions.
+
+Other parameters, like ``synchronous_node_count``, ``maximum_lag_on_syncnode``, and ``synchronous_mode_strict`` continue to work the same way as with ``synchronous_mode=on``.
+
+Example:
+---------
+
+``/config`` key in DCS
+^^^^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: YAML
+
+    synchronous_mode: quorum
+    synchronous_node_count: 2
+    ...
+
+``/sync`` key in DCS
+^^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: JSON
+
+    {
+        "leader": "node0",
+        "sync_standby": "node1,node2,node3",
+        "quorum": 1
+    }
+
+postgresql.conf
+^^^^^^^^^^^^^^^
+
+.. code-block:: INI
+
+    synchronous_standby_names = 'ANY 2 (node1,node2,node3)'
+
+
+If the primary (``node0``) failed, in the above example two of the ``node1``, ``node2``, ``node3`` will have the latest transaction received, but we don't know which ones. To figure out whether the node ``node1`` has received the latest transaction, we need to compare its LSN with the LSN on **at least** one node (``quorum=1`` in the ``/sync`` key) among ``node2`` and ``node3``. If ``node1`` isn't behind of at least one of them, we can guarantee that there will be no user visible data loss if ``node1`` is promoted.
 
 
 .. [1] The data is still there, but recovering it requires a manual recovery effort by data recovery specialists. When Patroni is allowed to rewind with ``use_pg_rewind`` the forked timeline will be automatically erased to rejoin the failed primary with the cluster.

docs/rest_api.rst

@@ -3,7 +3,7 @@
 Patroni REST API
 ================
 
-Patroni has a rich REST API, which is used by Patroni itself during the leader race, by the ``patronictl`` tool in order to perform failovers/switchovers/reinitialize/restarts/reloads, by HAProxy or any other kind of load balancer to perform HTTP health checks, and of course could also be used for monitoring. Below you will find the list of Patroni REST API endpoints.
+Patroni has a rich REST API, which is used by Patroni itself during the leader race, by the :ref:`patronictl` tool in order to perform failovers/switchovers/reinitialize/restarts/reloads, by HAProxy or any other kind of load balancer to perform HTTP health checks, and of course could also be used for monitoring. Below you will find the list of Patroni REST API endpoints.
 
 Health check endpoints
 ----------------------
@@ -45,6 +45,10 @@ For all health check ``GET`` requests Patroni returns a JSON document with the s
 
 - ``GET /read-only-sync``: like the above endpoint, but also includes the primary.
 
+- ``GET /quorum``: returns HTTP status code **200** only when this Patroni node is listed as a quorum node in ``synchronous_standby_names`` on the primary.
+
+- ``GET /read-only-quorum``: like the above endpoint, but also includes the primary.
+
 - ``GET /asynchronous`` or ``GET /async``: returns HTTP status code **200** only when the Patroni node is running as an asynchronous standby.
 
 
@@ -92,26 +96,188 @@ Monitoring endpoint
 
 The ``GET /patroni`` is used by Patroni during the leader race. It also could be used by your monitoring system. The JSON document produced by this endpoint has the same structure as the JSON produced by the health check endpoints.
 
+**Example:** A healthy cluster
+
 .. code-block:: bash
 
     $ curl -s http://localhost:8008/patroni | jq .
     {
       "state": "running",
-      "postmaster_start_time": "2019-09-24 09:22:32.555 CEST",
+      "postmaster_start_time": "2023-08-18 11:03:37.966359+00:00",
       "role": "master",
-      "server_version": 110005,
+      "server_version": 150004,
-      "cluster_unlocked": false,
       "xlog": {
-        "location": 25624640
+        "location": 67395656
       },
-      "timeline": 3,
+      "timeline": 1,
-      "database_system_identifier": "6739877027151648096",
+      "replication": [
+        {
+          "usename": "replicator",
+          "application_name": "patroni2",
+          "client_addr": "10.89.0.6",
+          "state": "streaming",
+          "sync_state": "async",
+          "sync_priority": 0
+        },
+        {
+          "usename": "replicator",
+          "application_name": "patroni3",
+          "client_addr": "10.89.0.2",
+          "state": "streaming",
+          "sync_state": "async",
+          "sync_priority": 0
+        }
+      ],
+      "dcs_last_seen": 1692356718,
+      "tags": {
+        "clonefrom": true
+      },
+      "database_system_identifier": "7268616322854375442",
       "patroni": {
-        "version": "1.6.0",
+        "version": "3.1.0",
-        "scope": "batman"
+        "scope": "demo",
+        "name": "patroni1"
       }
     }
 
+**Example:** An unlocked cluster
+
+.. code-block:: bash
+
+    $ curl -s http://localhost:8008/patroni  | jq .
+    {
+      "state": "running",
+      "postmaster_start_time": "2023-08-18 11:09:08.615242+00:00",
+      "role": "replica",
+      "server_version": 150004,
+      "xlog": {
+        "received_location": 67419744,
+        "replayed_location": 67419744,
+        "replayed_timestamp": null,
+        "paused": false
+      },
+      "timeline": 1,
+      "replication": [
+        {
+          "usename": "replicator",
+          "application_name": "patroni2",
+          "client_addr": "10.89.0.6",
+          "state": "streaming",
+          "sync_state": "async",
+          "sync_priority": 0
+        },
+        {
+          "usename": "replicator",
+          "application_name": "patroni3",
+          "client_addr": "10.89.0.2",
+          "state": "streaming",
+          "sync_state": "async",
+          "sync_priority": 0
+        }
+      ],
+      "cluster_unlocked": true,
+      "dcs_last_seen": 1692356928,
+      "tags": {
+        "clonefrom": true
+      },
+      "database_system_identifier": "7268616322854375442",
+      "patroni": {
+        "version": "3.1.0",
+        "scope": "demo",
+        "name": "patroni1"
+      }
+    }
+
+**Example:** An unlocked cluster with :ref:`DCS failsafe mode <dcs_failsafe_mode>` enabled
+
+.. code-block:: bash
+
+    $ curl -s http://localhost:8008/patroni  | jq .
+    {
+      "state": "running",
+      "postmaster_start_time": "2023-08-18 11:09:08.615242+00:00",
+      "role": "replica",
+      "server_version": 150004,
+      "xlog": {
+        "location": 67420024
+      },
+      "timeline": 1,
+      "replication": [
+        {
+          "usename": "replicator",
+          "application_name": "patroni2",
+          "client_addr": "10.89.0.6",
+          "state": "streaming",
+          "sync_state": "async",
+          "sync_priority": 0
+        },
+        {
+          "usename": "replicator",
+          "application_name": "patroni3",
+          "client_addr": "10.89.0.2",
+          "state": "streaming",
+          "sync_state": "async",
+          "sync_priority": 0
+        }
+      ],
+      "cluster_unlocked": true,
+      "failsafe_mode_is_active": true,
+      "dcs_last_seen": 1692356928,
+      "tags": {
+        "clonefrom": true
+      },
+      "database_system_identifier": "7268616322854375442",
+      "patroni": {
+        "version": "3.1.0",
+        "scope": "demo",
+        "name": "patroni1"
+      }
+    }
+
+**Example:** A cluster with the :ref:`pause mode <pause>` enabled
+
+.. code-block:: bash
+
+    $ curl -s http://localhost:8008/patroni  | jq .
+    {
+      "state": "running",
+      "postmaster_start_time": "2023-08-18 11:09:08.615242+00:00",
+      "role": "replica",
+      "server_version": 150004,
+      "xlog": {
+        "location": 67420024
+      },
+      "timeline": 1,
+      "replication": [
+        {
+          "usename": "replicator",
+          "application_name": "patroni2",
+          "client_addr": "10.89.0.6",
+          "state": "streaming",
+          "sync_state": "async",
+          "sync_priority": 0
+        },
+        {
+          "usename": "replicator",
+          "application_name": "patroni3",
+          "client_addr": "10.89.0.2",
+          "state": "streaming",
+          "sync_state": "async",
+          "sync_priority": 0
+        }
+      ],
+      "pause": true,
+      "dcs_last_seen": 1692356928,
+      "tags": {
+        "clonefrom": true
+      },
+      "database_system_identifier": "7268616322854375442",
+      "patroni": {
+        "version": "3.1.0",
+        "scope": "demo",
+        "name": "patroni1"
+      }
+    }
 
 Retrieve the Patroni metrics in Prometheus format through the ``GET /metrics`` endpoint.
 
@@ -121,64 +287,73 @@ Retrieve the Patroni metrics in Prometheus format through the ``GET /metrics`` e
 	
 	# HELP patroni_version Patroni semver without periods. \
 	# TYPE patroni_version gauge
-	patroni_version{scope="batman"} 020103
+	patroni_version{scope="batman",name="patroni1"} 020103
 	# HELP patroni_postgres_running Value is 1 if Postgres is running, 0 otherwise.
 	# TYPE patroni_postgres_running gauge
-	patroni_postgres_running{scope="batman"} 1
+	patroni_postgres_running{scope="batman",name="patroni1"} 1
 	# HELP patroni_postmaster_start_time Epoch seconds since Postgres started.
 	# TYPE patroni_postmaster_start_time gauge
-	patroni_postmaster_start_time{scope="batman"} 1657656955.179243
+	patroni_postmaster_start_time{scope="batman",name="patroni1"} 1657656955.179243
 	# HELP patroni_master Value is 1 if this node is the leader, 0 otherwise.
 	# TYPE patroni_master gauge
-	patroni_master{scope="batman"} 1
+	patroni_master{scope="batman",name="patroni1"} 1
+	# HELP patroni_primary Value is 1 if this node is the leader, 0 otherwise.
+	# TYPE patroni_primary gauge
+	patroni_primary{scope="batman",name="patroni1"} 1
 	# HELP patroni_xlog_location Current location of the Postgres transaction log, 0 if this node is not the leader.
 	# TYPE patroni_xlog_location counter
-	patroni_xlog_location{scope="batman"} 22320573386952
+	patroni_xlog_location{scope="batman",name="patroni1"} 22320573386952
 	# HELP patroni_standby_leader Value is 1 if this node is the standby_leader, 0 otherwise.
 	# TYPE patroni_standby_leader gauge
-	patroni_standby_leader{scope="batman"} 0
+	patroni_standby_leader{scope="batman",name="patroni1"} 0
 	# HELP patroni_replica Value is 1 if this node is a replica, 0 otherwise.
 	# TYPE patroni_replica gauge
-	patroni_replica{scope="batman"} 0
+	patroni_replica{scope="batman",name="patroni1"} 0
 	# HELP patroni_sync_standby Value is 1 if this node is a sync standby replica, 0 otherwise.
 	# TYPE patroni_sync_standby gauge
-	patroni_sync_standby{scope="batman"} 0
+	patroni_sync_standby{scope="batman",name="patroni1"} 0
+	# HELP patroni_quorum_standby Value is 1 if this node is a quorum standby replica, 0 otherwise.
+	# TYPE patroni_quorum_standby gauge
+	patroni_quorum_standby{scope="batman",name="patroni1"} 0
 	# HELP patroni_xlog_received_location Current location of the received Postgres transaction log, 0 if this node is not a replica.
 	# TYPE patroni_xlog_received_location counter
-	patroni_xlog_received_location{scope="batman"} 0
+	patroni_xlog_received_location{scope="batman",name="patroni1"} 0
 	# HELP patroni_xlog_replayed_location Current location of the replayed Postgres transaction log, 0 if this node is not a replica.
 	# TYPE patroni_xlog_replayed_location counter
-	patroni_xlog_replayed_location{scope="batman"} 0
+	patroni_xlog_replayed_location{scope="batman",name="patroni1"} 0
 	# HELP patroni_xlog_replayed_timestamp Current timestamp of the replayed Postgres transaction log, 0 if null.
 	# TYPE patroni_xlog_replayed_timestamp gauge
-	patroni_xlog_replayed_timestamp{scope="batman"} 0
+	patroni_xlog_replayed_timestamp{scope="batman",name="patroni1"} 0
 	# HELP patroni_xlog_paused Value is 1 if the Postgres xlog is paused, 0 otherwise.
 	# TYPE patroni_xlog_paused gauge
-	patroni_xlog_paused{scope="batman"} 0
+	patroni_xlog_paused{scope="batman",name="patroni1"} 0
 	# HELP patroni_postgres_streaming Value is 1 if Postgres is streaming, 0 otherwise.
 	# TYPE patroni_postgres_streaming gauge
-	patroni_postgres_streaming{scope="batman"} 1
+	patroni_postgres_streaming{scope="batman",name="patroni1"} 1
 	# HELP patroni_postgres_in_archive_recovery Value is 1 if Postgres is replicating from archive, 0 otherwise.
 	# TYPE patroni_postgres_in_archive_recovery gauge
-	patroni_postgres_in_archive_recovery{scope="batman"} 0
+	patroni_postgres_in_archive_recovery{scope="batman",name="patroni1"} 0
 	# HELP patroni_postgres_server_version Version of Postgres (if running), 0 otherwise.
 	# TYPE patroni_postgres_server_version gauge
-	patroni_postgres_server_version {scope="batman"} 140004
+	patroni_postgres_server_version{scope="batman",name="patroni1"} 140004
 	# HELP patroni_cluster_unlocked Value is 1 if the cluster is unlocked, 0 if locked.
 	# TYPE patroni_cluster_unlocked gauge
-	patroni_cluster_unlocked{scope="batman"} 0
+	patroni_cluster_unlocked{scope="batman",name="patroni1"} 0
 	# HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise.
 	# TYPE patroni_postgres_timeline counter
-	patroni_postgres_timeline{scope="batman"} 24
+	patroni_failsafe_mode_is_active{scope="batman",name="patroni1"} 0
+	# HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise.
+	# TYPE patroni_postgres_timeline counter
+	patroni_postgres_timeline{scope="batman",name="patroni1"} 24
 	# HELP patroni_dcs_last_seen Epoch timestamp when DCS was last contacted successfully by Patroni.
 	# TYPE patroni_dcs_last_seen gauge
-	patroni_dcs_last_seen{scope="batman"} 1677658321
+	patroni_dcs_last_seen{scope="batman",name="patroni1"} 1677658321
 	# HELP patroni_pending_restart Value is 1 if the node needs a restart, 0 otherwise.
 	# TYPE patroni_pending_restart gauge
-	patroni_pending_restart{scope="batman"} 1
+	patroni_pending_restart{scope="batman",name="patroni1"} 1
 	# HELP patroni_is_paused Value is 1 if auto failover is disabled, 0 otherwise.
 	# TYPE patroni_is_paused gauge
-	patroni_is_paused{scope="batman"} 1
+	patroni_is_paused{scope="batman",name="patroni1"} 1
 
 
 Cluster status endpoints
@@ -192,24 +367,24 @@ Cluster status endpoints
     {
       "members": [
         {
-          "name": "postgresql0",
+          "name": "patroni1",
-          "host": "127.0.0.1",
-          "port": 5432,
           "role": "leader",
           "state": "running",
-          "api_url": "http://127.0.0.1:8008/patroni",
+          "api_url": "http://10.89.0.4:8008/patroni",
+          "host": "10.89.0.4",
+          "port": 5432,
           "timeline": 5,
           "tags": {
             "clonefrom": true
           }
         },
         {
-          "name": "postgresql1",
+          "name": "patroni2",
-          "host": "127.0.0.1",
-          "port": 5433,
           "role": "replica",
-          "state": "running",
+          "state": "streaming",
-          "api_url": "http://127.0.0.1:8009/patroni",
+          "api_url": "http://10.89.0.6:8008/patroni",
+          "host": "10.89.0.6",
+          "port": 5433,
           "timeline": 5,
           "tags": {
             "clonefrom": true
@@ -217,9 +392,11 @@ Cluster status endpoints
           "lag": 0
         }
       ],
+      "scope": "demo",
       "scheduled_switchover": {
-        "at": "2019-09-24T10:36:00+02:00",
+        "at": "2023-09-24T10:36:00+02:00",
-        "from": "postgresql0"
+        "from": "patroni1",
+        "to": "patroni3"
       }
     }
 
@@ -256,6 +433,7 @@ Cluster status endpoints
       ]
     ]
 
+.. _config_endpoint:
 
 Config endpoint
 ---------------
@@ -264,7 +442,7 @@ Config endpoint
 
 .. code-block:: bash
 
-	$ curl -s localhost:8008/config | jq .
+	$ curl -s http://localhost:8008/config | jq .
 	{
 	  "ttl": 30,
 	  "loop_wait": 10,
@@ -275,7 +453,6 @@ Config endpoint
 	    "use_pg_rewind": true,
 	    "parameters": {
 	      "hot_standby": "on",
-	      "wal_log_hints": "on",
 	      "wal_level": "hot_standby",
 	      "max_wal_senders": 5,
 	      "max_replication_slots": 5,
@@ -302,7 +479,6 @@ Config endpoint
 	    "use_pg_rewind": true,
 	    "parameters": {
 	      "hot_standby": "on",
-	      "wal_log_hints": "on",
 	      "wal_level": "hot_standby",
 	      "max_wal_senders": 5,
 	      "max_replication_slots": 5,
@@ -326,8 +502,9 @@ Let's check that the node processed this configuration. First of all it should s
 	    "location": 2197818976
 	  },
 	  "patroni": {
+	    "version": "1.0",
 	    "scope": "batman",
-	    "version": "1.0"
+	    "name": "patroni1"
 	  },
 	  "state": "running",
 	  "role": "master",
@@ -355,7 +532,6 @@ If you want to remove (reset) some setting just patch it with ``null``:
 	      "hot_standby": "on",
 	      "unix_socket_directories": ".",
 	      "wal_level": "hot_standby",
-	      "wal_log_hints": "on",
 	      "max_wal_senders": 5,
 	      "max_replication_slots": 5
 	    }
@@ -369,7 +545,7 @@ The above call removes ``postgresql.parameters.max_connections`` from the dynami
 .. code-block:: bash
 
 	$ curl -s -XPUT -d \
-		'{"maximum_lag_on_failover":1048576,"retry_timeout":10,"postgresql":{"use_slots":true,"use_pg_rewind":true,"parameters":{"hot_standby":"on","wal_log_hints":"on","wal_level":"hot_standby","unix_socket_directories":".","max_wal_senders":5}},"loop_wait":3,"ttl":20}' \
+		'{"maximum_lag_on_failover":1048576,"retry_timeout":10,"postgresql":{"use_slots":true,"use_pg_rewind":true,"parameters":{"hot_standby":"on","wal_level":"hot_standby","unix_socket_directories":".","max_wal_senders":5}},"loop_wait":3,"ttl":20}' \
 		http://localhost:8008/config | jq .
 	{
 	  "ttl": 20,
@@ -381,7 +557,6 @@ The above call removes ``postgresql.parameters.max_connections`` from the dynami
 	      "hot_standby": "on",
 	      "unix_socket_directories": ".",
 	      "wal_level": "hot_standby",
-	      "wal_log_hints": "on",
 	      "max_wal_senders": 5
 	    },
 	    "use_pg_rewind": true
@@ -393,41 +568,114 @@ The above call removes ``postgresql.parameters.max_connections`` from the dynami
 Switchover and failover endpoints
 ---------------------------------
 
-``POST /switchover`` or ``POST /failover``. These endpoints are very similar to each other. There are a couple of minor differences though:
+.. _switchover_api:
 
-1. The failover endpoint allows to perform a manual failover when there are no healthy nodes, but at the same time it will not allow you to schedule a switchover.
+Switchover
+^^^^^^^^^^
 
-2. The switchover endpoint is the opposite. It works only when the cluster is healthy (there is a leader) and allows to schedule a switchover at a given time.
+``/switchover`` endpoint only works when the cluster is healthy (there is a leader). It also allows to schedule a switchover at a given time.
 
+When calling ``/switchover`` endpoint a candidate can be specified but is not required, in contrast to ``/failover`` endpoint. If a candidate is not provided, all the eligible nodes of the cluster will participate in the leader race after the leader stepped down.
 
-In the JSON body of the ``POST`` request you must specify at least the ``leader`` or ``candidate`` fields and optionally the ``scheduled_at`` field if you want to schedule a switchover at a specific time.
+In the JSON body of the ``POST`` request you must specify the ``leader`` field. The ``candidate`` and the ``scheduled_at`` fields are optional and can be used to schedule a switchover at a specific time.
 
+Depending on the situation, requests might return different HTTP status codes and bodies. Status code **200** is returned when the switchover or failover successfully completed. If the switchover was successfully scheduled, Patroni will return HTTP status code **202**. In case something went wrong, the error status code (one of **400**, **412**, or **503**) will be returned with some details in the response body.
 
-Example: perform a failover to the specific node:
+``DELETE /switchover`` can be used to delete the currently scheduled switchover.
+
+**Example:** perform a switchover to any healthy standby
 
 .. code-block:: bash
 
-    $ curl -s http://localhost:8009/failover -XPOST -d '{"candidate":"postgresql1"}'
+	$ curl -s http://localhost:8008/switchover -XPOST -d '{"leader":"postgresql1"}'
-    Successfully failed over to "postgresql1"
+	Successfully switched over to "postgresql2"
 
 
-Example: schedule a switchover from the leader to any other healthy replica in the cluster at a specific time:
+**Example:** perform a switchover to a specific node
 
 .. code-block:: bash
 
-    $ curl -s http://localhost:8008/switchover -XPOST -d \
+	$ curl -s http://localhost:8008/switchover -XPOST -d \
-	    '{"leader":"postgresql0","scheduled_at":"2019-09-24T12:00+00"}'
+		'{"leader":"postgresql1","candidate":"postgresql2"}'
-    Switchover scheduled
+	Successfully switched over to "postgresql2"
 
 
-Depending on the situation the request might finish with a different HTTP status code and body. The status code **200** is returned when the switchover or failover successfully completed. If the switchover was successfully scheduled, Patroni will return HTTP status code **202**. In case something went wrong, the error status code (one of **400**, **412** or **503**) will be returned with some details in the response body. For more information please check the source code of ``patroni/api.py:do_POST_failover()`` method.
+**Example:** schedule a switchover from the leader to any other healthy standby in the cluster at a specific time.
 
-- ``DELETE /switchover``: delete the scheduled switchover
+.. code-block:: bash
 
-The ``POST /switchover`` and ``POST failover`` endpoints are used by ``patronictl switchover`` and ``patronictl failover``, respectively.
+	$ curl -s http://localhost:8008/switchover -XPOST -d \
-The ``DELETE /switchover`` is used by ``patronictl flush <cluster-name> switchover``.
+		'{"leader":"postgresql0","scheduled_at":"2019-09-24T12:00+00"}'
+	Switchover scheduled
 
 
+Failover
+^^^^^^^^
+
+``/failover`` endpoint can be used to perform a manual failover when there are no healthy nodes (e.g. to an asynchronous standby if all synchronous standbys are not healthy enough to promote). However there is no requirement for a cluster not to have leader - failover can also be run on a healthy cluster.
+
+In the JSON body of the ``POST`` request you must specify the ``candidate`` field. If the ``leader`` field is specified, a switchover is triggered instead.
+
+**Example:**
+
+.. code-block:: bash
+
+	$ curl -s http://localhost:8008/failover -XPOST -d '{"candidate":"postgresql1"}'
+	Successfully failed over to "postgresql1"
+
+.. warning::
+	:ref:`Be very careful <failover_healthcheck>` when using this endpoint, as this can cause data loss in certain situations. In most cases, :ref:`the switchover endpoint <switchover_api>` satisfies the administrator's needs. 
+
+
+``POST /switchover`` and ``POST /failover`` endpoints are used by :ref:`patronictl_switchover` and :ref:`patronictl_failover`, respectively.
+
+``DELETE /switchover`` is used by :ref:`patronictl flush cluster-name switchover <patronictl_flush_parameters>`.
+
+.. list-table:: Failover/Switchover comparison
+   :widths: 25 25 25
+   :header-rows: 1
+
+   * -
+     - Failover
+     - Switchover
+   * - Requires leader specified
+     - no
+     - yes
+   * - Requires candidate specified
+     - yes
+     - no
+   * - Can be run in pause
+     - yes
+     - yes (only to a specific candidate)
+   * - Can be scheduled
+     - no
+     - yes (if not in pause)
+
+.. _failover_healthcheck:
+
+Healthy standby
+^^^^^^^^^^^^^^^
+
+There are a couple of checks that a member of a cluster should pass to be able to participate in the leader race during a switchover or to become a leader as a failover/switchover candidate:
+
+- be reachable via Patroni API;
+- not have ``nofailover`` tag set to ``true``;
+- have watchdog fully functional (if required by the configuration);
+- in case of a switchover in a healthy cluster or an automatic failover, not exceed maximum replication lag (``maximum_lag_on_failover`` :ref:`configuration parameter <dynamic_configuration>`);
+- in case of a switchover in a healthy cluster or an automatic failover, not have a timeline number smaller than the cluster timeline if ``check_timeline`` :ref:`configuration parameter <dynamic_configuration>` is set to ``true``;
+- in :ref:`synchronous mode <synchronous_mode>`:
+
+  - In case of a switchover (both with and without a candidate): be listed in the ``/sync`` key members;
+  - For a failover in both healthy and unhealthy clusters, this check is omitted.
+
+.. warning::
+    In case of a manual failover in a cluster without a leader, a candidate will be allowed to promote even if:
+	- it is not in the ``/sync`` key members when synchronous mode is enabled;
+	- its lag exceeds the maximum replication lag allowed;
+	- it has the timeline number smaller than the last known cluster timeline.
+
+.. _restart_endpoint:
+
 Restart endpoint
 ----------------
 
@@ -441,15 +689,16 @@ Restart endpoint
 
 - ``DELETE /restart``: delete the scheduled restart
 
-``POST /restart`` and ``DELETE /restart`` endpoints are used by ``patronictl restart`` and ``patronictl flush <cluster-name> restart`` respectively.
+``POST /restart`` and ``DELETE /restart`` endpoints are used by :ref:`patronictl_restart` and :ref:`patronictl flush cluster-name restart <patronictl_flush_parameters>` respectively.
 
+.. _reload_endpoint:
 
 Reload endpoint
 ---------------
 
-The ``POST /reload`` call will order Patroni to re-read and apply the configuration file. This is the equivalent of sending the ``SIGHUP`` signal to the Patroni process. In case you changed some of the Postgres parameters which require a restart (like **shared_buffers**), you still have to explicitly do the restart of Postgres by either calling the ``POST /restart`` endpoint or with the help of ``patronictl restart``.
+The ``POST /reload`` call will order Patroni to re-read and apply the configuration file. This is the equivalent of sending the ``SIGHUP`` signal to the Patroni process. In case you changed some of the Postgres parameters which require a restart (like **shared_buffers**), you still have to explicitly do the restart of Postgres by either calling the ``POST /restart`` endpoint or with the help of :ref:`patronictl_restart`.
 
-The reload endpoint is used by ``patronictl reload``.
+The reload endpoint is used by :ref:`patronictl_reload`.
 
 
 Reinitialize endpoint
@@ -459,4 +708,4 @@ Reinitialize endpoint
 
 The call might fail if Patroni is in a loop trying to recover (restart) a failed Postgres. In order to overcome this problem one can specify ``{"force":true}`` in the request body.
 
-The reinitialize endpoint is used by ``patronictl reinit``.
+The reinitialize endpoint is used by :ref:`patronictl_reinit`.

docs/security.rst

@@ -9,7 +9,7 @@ A Patroni cluster has two interfaces to be protected from unauthorized access: t
 Protecting DCS
 ==============
 
-Patroni and patronictl both store and retrieve data to/from the DCS.
+Patroni and :ref:`patronictl` both store and retrieve data to/from the DCS.
 
 Despite DCS doesn't contain any sensitive information, it allows changing some of Patroni/Postgres configuration. Therefore the very first thing that should be protected is DCS itself.
 
@@ -22,7 +22,7 @@ Protecting the REST API
 
 Protecting the REST API is a more complicated task.
 
-The Patroni REST API is used by Patroni itself during the leader race, by the ``patronictl`` tool in order to perform failovers/switchovers/reinitialize/restarts/reloads, by HAProxy or any other kind of load balancer to perform HTTP health checks, and of course could also be used for monitoring.
+The Patroni REST API is used by Patroni itself during the leader race, by the :ref:`patronictl` tool in order to perform failovers/switchovers/reinitialize/restarts/reloads, by HAProxy or any other kind of load balancer to perform HTTP health checks, and of course could also be used for monitoring.
 
 From the point of view of security, REST API contains safe (``GET`` requests, only retrieve information) and unsafe (``PUT``, ``POST``, ``PATCH`` and ``DELETE`` requests, change the state of nodes) endpoints.
 
@@ -32,6 +32,6 @@ When TLS for the REST API is enabled and a PKI is established, mutual authentica
 
 The ``restapi`` section parameters enable TLS client authentication to the server. Depending on the value of the ``verify_client`` parameter, the API server requires a successful client certificate verification for both safe and unsafe API calls (``verify_client: required``), or only for unsafe API calls (``verify_client: optional``), or for no API calls (``verify_client: none``).
 
-The ``ctl`` section parameters enable TLS server authentication to the client (the ``patronictl`` tool which uses the same config as patroni). Set ``insecure: true`` to disable the server certificate verification by the client. See :ref:`settings <patronictl_settings>` for a detailed description of the TLS client parameters.
+The ``ctl`` section parameters enable TLS server authentication to the client (the :ref:`patronictl` tool which uses the same config as patroni). Set ``insecure: true`` to disable the server certificate verification by the client. See :ref:`settings <patronictl_settings>` for a detailed description of the TLS client parameters.
 
 Protecting the PostgreSQL database proper from unauthorized access is beyond the scope of this document and is covered in https://www.postgresql.org/docs/current/client-authentication.html

docs/standby_cluster.rst

@@ -0,0 +1,78 @@
+.. _standby_cluster:
+
+Standby cluster
+---------------
+
+Patroni also support running cascading replication to a remote datacenter
+(region) using a feature that is called "standby cluster". This type of
+clusters has:
+
+* "standby leader", that behaves pretty much like a regular cluster leader,
+  except it replicates from a remote node.
+
+* cascade replicas, that are replicating from standby leader.
+
+Standby leader holds and updates a leader lock in DCS. If the leader lock
+expires, cascade replicas will perform an election to choose another leader
+from the standbys.
+
+There is no further relationship between the standby cluster and the primary
+cluster it replicates from, in particular, they must not share the same DCS
+scope if they use the same DCS. They do not know anything else from each other
+apart from replication information. Also, the standby cluster is not being
+displayed in :ref:`patronictl_list` or :ref:`patronictl_topology` output on the
+primary cluster.
+
+For the sake of flexibility, you can specify methods of creating a replica and
+recovery WAL records when a cluster is in the "standby mode" by providing
+:ref:`create_replica_methods <custom_replica_creation>` key in
+`standby_cluster` section. It is distinct from creating replicas, when cluster
+is detached and functions as a normal cluster, which is controlled by
+`create_replica_methods` in `postgresql` section. Both "standby" and "normal"
+`create_replica_methods` reference  keys in `postgresql` section.
+
+To configure such cluster you need to specify the section ``standby_cluster``
+in a patroni configuration:
+
+.. code:: YAML
+
+    bootstrap:
+        dcs:
+            standby_cluster:
+                host: 1.2.3.4
+                port: 5432
+                primary_slot_name: patroni
+                create_replica_methods:
+                - basebackup
+
+Note, that these options will be applied only once during cluster bootstrap,
+and the only way to change them afterwards is through DCS.
+
+Patroni expects to find `postgresql.conf` or `postgresql.conf.backup` in PGDATA
+of the remote primary and will not start if it does not find it after a
+basebackup. If the remote primary keeps its `postgresql.conf` elsewhere, it is
+your responsibility to copy it to PGDATA.
+
+If you use replication slots on the standby cluster, you must also create the
+corresponding replication slot on the primary cluster.  It will not be done
+automatically by the standby cluster implementation.  You can use Patroni's
+permanent replication slots feature on the primary cluster to maintain a
+replication slot with the same name as ``primary_slot_name``, or its default
+value if ``primary_slot_name`` is not provided.
+
+In case the remote site doesn't provide a single endpoint that connects to a
+primary, one could list all hosts of the source cluster in the
+``standby_cluster.host`` section.  When ``standby_cluster.host`` contains
+multiple hosts separated by commas, Patroni will:
+
+* add ``target_session_attrs=read-write`` to the ``primary_conninfo`` on the
+  standby leader node.
+* use ``target_session_attrs=read-write`` when trying to determine whether we
+  need to run ``pg_rewind`` or when executing ``pg_rewind`` on all nodes of the
+  standby cluster.
+  
+There is also a possibility to replicate the standby cluster from another
+standby cluster or from a standby member of the primary cluster: for that, you
+need to define a single host in the ``standby_cluster.host`` section. However,
+you need to beware that in this case ``pg_rewind`` will fail to execute on the
+standby cluster.

docs/tools_integration.rst

@@ -0,0 +1,62 @@
+Integration with other tools
+============================
+
+Patroni is able to integrate with other tools in your stack. In this section you
+will find a list of examples, which although not an exhaustive list, might
+provide you with ideas on how Patroni can integrate with other tools.
+
+Barman
+------
+
+Patroni delivers an application named ``patroni_barman`` which has logic to
+communicate with ``pg-backup-api``, so you are able to perform Barman operations
+remotely.
+
+This application currently has a couple of sub-commands: ``recover`` and
+``config-switch``.
+
+patroni_barman recover
+^^^^^^^^^^^^^^^^^^^^^^
+
+The ``recover`` sub-command can be used as a custom bootstrap or custom replica
+creation method. You can find more information about that in
+:ref:`replica_imaging_and_bootstrap`.
+
+patroni_barman config-switch
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The ``config-switch`` sub-command is designed to be used as an ``on_role_change``
+callback in Patroni. As an example, assume you are streaming WALs from your
+current primary to your Barman host. In the event of a failover in the cluster
+you might want to start streaming WALs from the new primary. You can accomplish
+this by using ``patroni_barman config-switch`` as the ``on_role_change`` callback.
+
+.. note::
+    That sub-command relies on the ``barman config-switch`` command, which is in
+    charge of overriding the configuration of a Barman server by applying a
+    pre-defined model on top of it. This command is available since Barman 3.10.
+    Please consult the Barman documentation for more details.
+
+This is an example of how you can configure Patroni to apply a configuration
+model in case this Patroni node is promoted to primary:
+
+.. code:: YAML
+
+    postgresql:
+        callbacks:
+            on_role_change: >
+                patroni_barman
+                    --api-url YOUR_API_URL
+                    config-switch
+                    --barman-server YOUR_BARMAN_SERVER_NAME
+                    --barman-model YOUR_BARMAN_MODEL_NAME
+                    --switch-when promoted
+
+.. note::
+    ``patroni_barman config-switch`` requires that you have both Barman and
+    ``pg-backup-api`` configured in the Barman host, so it can execute a remote
+    ``barman config-switch`` through the backup API. Also, it requires that you
+    have pre-configured Barman models to be applied. The above example uses a
+    subset of the available parameters. You can get more information running
+    ``patroni_barman config-switch --help``, and by consulting the Barman
+    documentation.

docs/yaml_configuration.rst

@@ -11,12 +11,22 @@ Global/Universal
 -  **namespace**: path within the configuration store where Patroni will keep information about the cluster. Default value: "/service"
 -  **scope**: cluster name
 
+.. _log_settings:
+
 Log
 ---
+-  **type**: sets the format of logs. Can be either **plain** or **json**. To use **json** format, you must have the :ref:`jsonlogger <extras>` installed. The default value is **plain**.
 -  **level**: sets the general logging level. Default value is **INFO** (see `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>`_)
 -  **traceback\_level**: sets the level where tracebacks will be visible. Default value is **ERROR**. Set it to **DEBUG** if you want to see tracebacks only if you enable **log.level=DEBUG**.
--  **format**: sets the log formatting string. Default value is **%(asctime)s %(levelname)s: %(message)s** (see `the LogRecord attributes <https://docs.python.org/3.6/library/logging.html#logrecord-attributes>`_)
+-  **format**: sets the log formatting string. If the log type is **plain**, the log format should be a string. Refer to
+   `the LogRecord attributes <https://docs.python.org/3.6/library/logging.html#logrecord-attributes>`_ for
+   available attributes. If the log type is **json**, the log format can be a list in addition to a string. Each list
+   item should correspond to LogRecord attributes. Be cautious that only the field name is required, and the **%(**
+   and **)** should be omitted. If you wish to print a log field with a different key name, use a dictionary where
+   the dictionary key is the log field, and the value is the name of the field you want to be printed in the log.
+   Default value is **%(asctime)s %(levelname)s: %(message)s**
 -  **dateformat**: sets the datetime formatting string. (see the `formatTime() documentation <https://docs.python.org/3.6/library/logging.html#logging.Formatter.formatTime>`_)
+-  **static_fields**: add additional fields to the log. This option is only available when the log type is set to **json**.
 -  **max\_queue\_size**: Patroni is using two-step logging. Log records are written into the in-memory queue and there is a separate thread which pulls them from the queue and writes to stderr or file. The maximum size of the internal queue is limited by default by **1000** records, which is enough to keep logs for the past 1h20m.
 -  **dir**: Directory to write application logs to. The directory must exist and be writable by the user executing Patroni. If you set this value, the application will retain 4 25MB logs by default. You can tune those retention values with `file_num` and `file_size` (see below).
 -  **file\_num**: The number of application logs to retain.
@@ -26,13 +36,33 @@ Log
    -  **patroni.postmaster: WARNING**
    -  **urllib3: DEBUG**
 
+Here is an example of how to config patroni to log in json format.
+
+.. code:: YAML
+
+   log:
+      type: json
+      format:
+         - message
+         - module
+         - asctime: '@timestamp'
+         - levelname: level
+      static_fields:
+         app: patroni
+
 .. _bootstrap_settings:
 
 Bootstrap configuration
 -----------------------
+
+.. note::
+    Once Patroni has initialized the cluster for the first time and settings have been stored in the DCS, all future
+    changes to the ``bootstrap.dcs`` section of the YAML configuration will not take any effect! If you want to change
+    them please use either :ref:`patronictl_edit_config` or the Patroni :ref:`REST API <rest_api>`.
+
 -  **bootstrap**:
 
-   -  **dcs**: This section will be written into `/<namespace>/<scope>/config` of the given configuration store after initializing of new cluster. The global dynamic configuration for the cluster. Under the ``bootstrap.dcs`` you can put any of the parameters described in the :ref:`Dynamic Configuration settings <dynamic_configuration>` and after Patroni initialized (bootstrapped) the new cluster, it will write this section into `/<namespace>/<scope>/config` of the configuration store. All later changes of ``bootstrap.dcs`` will not take any effect! If you want to change them please use either ``patronictl edit-config`` or Patroni :ref:`REST API <rest_api>`.
+   -  **dcs**: This section will be written into `/<namespace>/<scope>/config` of the given configuration store after initializing the new cluster. The global dynamic configuration for the cluster. You can put any of the parameters described in the :ref:`Dynamic Configuration settings <dynamic_configuration>` under ``bootstrap.dcs`` and after Patroni has initialized (bootstrapped) the new cluster, it will write this section into `/<namespace>/<scope>/config` of the configuration store.
    -  **method**: custom script to use for bootstrapping this cluster.
 
       See :ref:`custom bootstrap methods documentation <custom_bootstrap>` for details.
@@ -43,15 +73,6 @@ Bootstrap configuration
       -  **- data-checksums**: Must be enabled when pg_rewind is needed on 9.3.
       -  **- encoding: UTF8**: default encoding for new databases.
       -  **- locale: UTF8**: default locale for new databases.
-   -  **users**: Some additional users which need to be created after initializing new cluster
-
-      -  **admin**: the name of user
-
-         -  **password**: (optional) password for the user
-         -  **options**: list of options for CREATE USER statement
-
-            -  **- createrole**
-            -  **- createdb**
    -  **post\_bootstrap** or **post\_init**: An additional script that will be executed after initializing the cluster. The script receives a connection string URL (with the cluster superuser as a user name). The PGPASSFILE variable is set to the location of pgpass file.
 
 .. _citus_settings:
@@ -136,6 +157,7 @@ ZooKeeper
 -  **key_password**: (optional) The client key password.
 -  **verify**: (optional) Whether to verify certificate or not. Defaults to ``true``.
 -  **set_acls**: (optional) If set, configure Kazoo to apply a default ACL to each ZNode that it creates. ACLs will assume 'x509' schema and should be specified as a dictionary with the principal as the key and one or more permissions as a list in the value.  Permissions may be one of ``CREATE``, ``READ``, ``WRITE``, ``DELETE`` or ``ADMIN``.  For example, ``set_acls: {CN=principal1: [CREATE, READ], CN=principal2: [ALL]}``.
+-  **auth_data**: (optional) Authentication credentials to use for the connection. Should be a dictionary in the form that `scheme` is the key and `credential` is the value. Defaults to empty dictionary.
 
 .. note::
     It is required to install ``kazoo>=2.6.0`` to support SSL.
@@ -155,7 +177,11 @@ Kubernetes
 -  **namespace**: (optional) Kubernetes namespace where Patroni pod is running. Default value is `default`.
 -  **labels**: Labels in format ``{label1: value1, label2: value2}``. These labels will be used to find existing objects (Pods and either Endpoints or ConfigMaps) associated with the current cluster. Also Patroni will set them on every object (Endpoint or ConfigMap) it creates.
 -  **scope\_label**: (optional) name of the label containing cluster name. Default value is `cluster-name`.
--  **role\_label**: (optional) name of the label containing role (master or replica). Patroni will set this label on the pod it runs in. Default value is ``role``.
+-  **role\_label**: (optional) name of the label containing role (master or replica or other custom value). Patroni will set this label on the pod it runs in. Default value is ``role``.
+-  **leader\_label\_value**: (optional) value of the pod label when Postgres role is ``master``. Default value is ``master``.
+-  **follower\_label\_value**: (optional) value of the pod label when Postgres role is ``replica``. Default value is ``replica``.
+-  **standby\_leader\_label\_value**: (optional) value of the pod label when Postgres role is ``standby_leader``. Default value is ``master``.
+-  **tmp_\role\_label**: (optional) name of the temporary label containing role (master or replica). Value of this label will always use the default of corresponding role. Set only when necessary.
 -  **use\_endpoints**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
 -  **pod\_ip**: (optional) IP address of the pod Patroni is running in. This value is required when `use_endpoints` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
 -  **ports**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``kubernetes.ports: [{"name": "postgresql", "port": 5432}]`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `kubernetes.use_endpoints` is set.
@@ -335,17 +361,27 @@ Here is an example of both **http_extra_headers** and **https_extra_headers**:
           https_extra_headers:
             'Strict-Transport-Security': 'max-age=31536000; includeSubDomains'
 
+.. warning::
+
+    - The ``restapi.connect_address`` must be accessible from all nodes of a given Patroni cluster. Internally Patroni is using it during the leader race to find nodes with minimal replication lag.
+    - If you enabled client certificates validation (``restapi.verify_client`` is set to ``required``), you also **must** provide **valid client certificates** in the ``ctl.certfile``, ``ctl.keyfile``, ``ctl.keyfile_password``. If not provided, Patroni will not work correctly.
+
+
 .. _patronictl_settings:
 
 CTL
 ---
 -  **ctl**: (optional)
 
+   -  **authentication**:
+
+      -  **username**: Basic-auth username for accessing protected REST API endpoints. If not provided :ref:`patronictl` will use the value provided for REST API "username" parameter.
+      -  **password**: Basic-auth password for accessing protected REST API endpoints. If not provided :ref:`patronictl` will use the value provided for REST API "password" parameter.
    -  **insecure**: Allow connections to REST API without verifying SSL certs.
-   -  **cacert**: Specifies the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs. If not provided patronictl will use the value provided for REST API "cafile" parameter.
+   -  **cacert**: Specifies the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs. If not provided :ref:`patronictl` will use the value provided for REST API "cafile" parameter.
-   -  **certfile**: Specifies the file with the client certificate in the PEM format. If not provided patronictl will use the value provided for REST API "certfile" parameter.
+   -  **certfile**: Specifies the file with the client certificate in the PEM format.
-   -  **keyfile**: Specifies the file with the client secret key in the PEM format. If not provided patronictl will use the value provided for REST API "keyfile" parameter.
+   -  **keyfile**: Specifies the file with the client secret key in the PEM format.
-   -  **keyfile\_password**: Specifies a password for decrypting the keyfile. If not provided patronictl will use the value provided for REST API "keyfile\_password" parameter.
+   -  **keyfile\_password**: Specifies a password for decrypting the client keyfile.
 
 Watchdog
 --------
@@ -357,11 +393,16 @@ Watchdog
 
 Tags
 ----
--  **nofailover**: ``true`` or ``false``, controls whether this node is allowed to participate in the leader race and become a leader. Defaults to ``false``
 -  **clonefrom**: ``true`` or ``false``. If set to ``true`` other nodes might prefer to use this node for bootstrap (take ``pg_basebackup`` from). If there are several nodes with ``clonefrom`` tag set to ``true`` the node to bootstrap from will be chosen randomly. The default value is ``false``.
 -  **noloadbalance**: ``true`` or ``false``. If set to ``true`` the node will return HTTP Status Code 503 for the ``GET /replica`` REST API health-check and therefore will be excluded from the load-balancing. Defaults to ``false``.
 -  **replicatefrom**: The IP address/hostname of another replica. Used to support cascading replication.
 -  **nosync**: ``true`` or ``false``. If set to ``true`` the node will never be selected as a synchronous replica.
+-  **nofailover**: ``true`` or ``false``, controls whether this node is allowed to participate in the leader race and become a leader. Defaults to ``false``, meaning this node _can_ participate in leader races. 
+-  **failover_priority**: integer, controls the priority that this node should have during failover. Nodes with higher priority will be preferred over lower priority nodes if they received/replayed the same amount of WAL. However, nodes with higher values of receive/replay LSN are preferred regardless of their priority. If the ``failover_priority`` is 0 or negative - such node is not allowed to participate in the leader race and to become a leader (similar to ``nofailover: true``).
+-  **nostream**: ``true`` or ``false``. If set to ``true`` the node will not use replication protocol to stream WAL. It will rely instead on archive recovery (if ``restore_command`` is configured) and ``pg_wal``/``pg_xlog`` polling. It also disables copying and synchronization of permanent logical replication slots on the node itself and all its cascading replicas. Setting this tag on primary node has no effect.
+
+.. warning::
+   Provide only one of ``nofailover`` or ``failover_priority``. Providing ``nofailover: true`` is the same as ``failover_priority: 0``, and providing ``nofailover: false`` will give the node priority 1. 
 
 In addition to these predefined tags, you can also add your own ones:
 
@@ -370,4 +411,4 @@ In addition to these predefined tags, you can also add your own ones:
 -  **key3**: ``1.4``
 -  **key4**: ``"RandomString"``
 
-Tags are visible in the :ref:`REST API <rest_api>` and ``patronictl list`` You can also check for an instance health using these tags. If the tag isn't defined for an instance, or if the respective value doesn't match the querying value, it will return HTTP Status Code 503.
+Tags are visible in the :ref:`REST API <rest_api>` and :ref:`patronictl_list` You can also check for an instance health using these tags. If the tag isn't defined for an instance, or if the respective value doesn't match the querying value, it will return HTTP Status Code 503.

features/Dockerfile

@@ -27,8 +27,8 @@ RUN set -ex \
     && apt-get update \
     && apt-get reinstall init-system-helpers \
     && apt-get install -y \
-      python3-pip \
       python3-dev \
+      python3-venv \
       rsync \
       curl \
       gcc \
@@ -40,7 +40,9 @@ RUN set -ex \
       net-tools \
       iputils-ping \
     && rm -rf /var/cache/apt \
-    && python3 -m pip install --no-cache-dir tox \
+    \
+    && python3 -m venv /tox \
+    && /tox/bin/pip install --no-cache-dir tox>=4 \
     \
     && mkdir -p "$PGHOME" \
     && sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \
@@ -50,6 +52,7 @@ RUN set -ex \
     && curl -sL "$ETCDURL/etcd-v$ETCDVERSION-linux-$(dpkg --print-architecture).tar.gz" \
        | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl
 
+ENV PATH="/tox/bin:$PATH"
 
 # This Dockerfile syntax only works with docker buildx and the syntax
 # line at the top of this file.

features/backup_restore.py

@@ -6,6 +6,7 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--datadir", required=True)
     parser.add_argument("--sourcedir", required=True)
+    parser.add_argument("--test-argument", required=True)
     args, _ = parser.parse_known_args()
 
     shutil.copytree(args.sourcedir, args.datadir)

features/basic_replication.feature

@@ -79,6 +79,7 @@ Feature: basic replication
     When I add the table buz to postgres2
     Then table buz is present on postgres0 after 20 seconds
 
+  @reject-duplicate-name
   Scenario: check graceful rejection when two nodes have the same name
     Given I start duplicate postgres0 on port 8011
-    Then there is a "Can't start; there is already a node named 'postgres0' running" CRITICAL in the dup-postgres0 patroni log
+    Then there is one of ["Can't start; there is already a node named 'postgres0' running"] CRITICAL in the dup-postgres0 patroni log after 5 seconds

features/citus.feature

@@ -68,6 +68,6 @@ Feature: citus
     And I receive a response output "+ttl: 20"
     Then postgres4 is registered in the postgres2 as the primary in group 2 after 5 seconds
     When I shut down postgres4
-    Then There is a transaction in progress on postgres0 changing pg_dist_node
+    Then there is a transaction in progress on postgres0 changing pg_dist_node after 5 seconds
     When I run patronictl.py restart batman postgres2 --group 1 --force
     Then a transaction finishes in 20 seconds

features/dcs_failsafe_mode.feature

@@ -4,14 +4,14 @@ Feature: dcs failsafe mode
   Scenario: check failsafe mode can be successfully enabled
     Given I start postgres0
     And postgres0 is a leader after 10 seconds
-    And I sleep for 3 seconds
+    Then "config" key in DCS has ttl=30 after 10 seconds
-    When I issue a PATCH request to http://127.0.0.1:8008/config with {"loop_wait": 2, "ttl": 20, "retry_timeout": 5, "failsafe_mode": true}
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"loop_wait": 2, "ttl": 20, "retry_timeout": 3, "failsafe_mode": true}
     Then I receive a response code 200
     And Response on GET http://127.0.0.1:8008/failsafe contains postgres0 after 10 seconds
     When I issue a GET request to http://127.0.0.1:8008/failsafe
     Then I receive a response code 200
     And I receive a response postgres0 http://127.0.0.1:8008/patroni
-    When I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"wal_level": "logical"}}}
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"wal_level": "logical"}},"slots":{"dcs_slot_1": null,"postgres0":null}}
     Then I receive a response code 200
     When I issue a PATCH request to http://127.0.0.1:8008/config with {"slots": {"dcs_slot_0": {"type": "logical", "database": "postgres", "plugin": "test_decoding"}}}
     Then I receive a response code 200
@@ -28,7 +28,6 @@ Feature: dcs failsafe mode
     When I do a backup of postgres0
     And I shut down postgres0
     When I start postgres1 in a cluster batman from backup with no_leader
-    And I sleep for 2 seconds
     Then postgres1 role is the replica after 12 seconds
 
   Scenario: check leader and replica are both in /failsafe key after leader is back
@@ -45,41 +44,73 @@ Feature: dcs failsafe mode
   @dcs-failsafe
   @slot-advance
   Scenario: check leader and replica are functioning while DCS is down
-    Given logical slot dcs_slot_0 is in sync between postgres0 and postgres1 after 10 seconds
+    Given I get all changes from physical slot dcs_slot_1 on postgres0
+    Then physical slot dcs_slot_1 is in sync between postgres0 and postgres1 after 10 seconds
+    And logical slot dcs_slot_0 is in sync between postgres0 and postgres1 after 10 seconds
     And DCS is down
     Then Response on GET http://127.0.0.1:8008/primary contains failsafe_mode_is_active after 12 seconds
     Then postgres0 role is the primary after 10 seconds
     And postgres1 role is the replica after 2 seconds
     And replication works from postgres0 to postgres1 after 10 seconds
-    And I get all changes from logical slot dcs_slot_0 on postgres0
+    When I get all changes from logical slot dcs_slot_0 on postgres0
-    And logical slot dcs_slot_0 is in sync between postgres0 and postgres1 after 20 seconds
+    And I get all changes from physical slot dcs_slot_1 on postgres0
+    Then logical slot dcs_slot_0 is in sync between postgres0 and postgres1 after 20 seconds
+    And physical slot dcs_slot_1 is in sync between postgres0 and postgres1 after 10 seconds
 
   @dcs-failsafe
   Scenario: check primary is demoted when one replica is shut down and DCS is down
     Given DCS is down
     And I kill postgres1
     And I kill postmaster on postgres1
-    And I sleep for 2 seconds
     Then postgres0 role is the replica after 12 seconds
 
   @dcs-failsafe
   Scenario: check known replica is promoted when leader is down and DCS is up
-    Given I shut down postgres0
+    Given I kill postgres0
+    And I shut down postmaster on postgres0
     And DCS is up
     When I start postgres1
     Then "members/postgres1" key in DCS has state=running after 10 seconds
     And postgres1 role is the primary after 25 seconds
 
   @dcs-failsafe
-  Scenario: check three-node cluster is functioning while DCS is down
+  Scenario: scale to three-node cluster
     Given I start postgres0
     And I start postgres2
     Then "members/postgres2" key in DCS has state=running after 10 seconds
     And "members/postgres0" key in DCS has state=running after 20 seconds
     And Response on GET http://127.0.0.1:8008/failsafe contains postgres2 after 10 seconds
     And replication works from postgres1 to postgres0 after 10 seconds
+    And replication works from postgres1 to postgres2 after 10 seconds
+
+  @dcs-failsafe
+  @slot-advance
+  Scenario: make sure permanent slots exist on replicas
+    Given I issue a PATCH request to http://127.0.0.1:8009/config with {"slots":{"dcs_slot_0":null,"dcs_slot_2":{"type":"logical","database":"postgres","plugin":"test_decoding"}}}
+    Then logical slot dcs_slot_2 is in sync between postgres1 and postgres0 after 20 seconds
+    And logical slot dcs_slot_2 is in sync between postgres1 and postgres2 after 20 seconds
+    When I get all changes from physical slot dcs_slot_1 on postgres1
+    Then physical slot dcs_slot_1 is in sync between postgres1 and postgres0 after 10 seconds
+    And physical slot dcs_slot_1 is in sync between postgres1 and postgres2 after 10 seconds
+    And physical slot postgres0 is in sync between postgres1 and postgres2 after 10 seconds
+
+  @dcs-failsafe
+  Scenario: check three-node cluster is functioning while DCS is down
     Given DCS is down
-    Then Response on GET http://127.0.0.1:8008/primary contains failsafe_mode_is_active after 12 seconds
+    Then Response on GET http://127.0.0.1:8009/primary contains failsafe_mode_is_active after 12 seconds
     Then postgres1 role is the primary after 10 seconds
     And postgres0 role is the replica after 2 seconds
     And postgres2 role is the replica after 2 seconds
+
+  @dcs-failsafe
+  @slot-advance
+  Scenario: check that permanent slots are in sync between nodes while DCS is down
+    Given replication works from postgres1 to postgres0 after 10 seconds
+    And replication works from postgres1 to postgres2 after 10 seconds
+    When I get all changes from logical slot dcs_slot_2 on postgres1
+    And I get all changes from physical slot dcs_slot_1 on postgres1
+    Then logical slot dcs_slot_2 is in sync between postgres1 and postgres0 after 20 seconds
+    And logical slot dcs_slot_2 is in sync between postgres1 and postgres2 after 20 seconds
+    And physical slot dcs_slot_1 is in sync between postgres1 and postgres0 after 10 seconds
+    And physical slot dcs_slot_1 is in sync between postgres1 and postgres2 after 10 seconds
+    And physical slot postgres0 is in sync between postgres1 and postgres2 after 10 seconds

features/environment.py

@@ -59,7 +59,7 @@ class AbstractController(abc.ABC):
                 break
             time.sleep(1)
         else:
-            assert False,\
+            assert False, \
                 "{0} instance is not available for queries after {1} seconds".format(self._name, max_wait_limit)
 
     def stop(self, kill=False, timeout=15, _=False):
@@ -162,9 +162,10 @@ class PatroniController(AbstractController):
 
     def stop(self, kill=False, timeout=15, postgres=False):
         if postgres:
-            return subprocess.call(['pg_ctl', '-D', self._data_dir, 'stop', '-mi', '-w'])
+            mode = 'i' if kill else 'f'
+            return subprocess.call(['pg_ctl', '-D', self._data_dir, 'stop', '-m' + mode, '-w'])
         super(PatroniController, self).stop(kill, timeout)
-        if isinstance(self._context.dcs_ctl, KubernetesController):
+        if isinstance(self._context.dcs_ctl, KubernetesController) and not kill:
             self._context.dcs_ctl.delete_pod(self._name[8:])
         if self.watchdog:
             self.watchdog.stop()
@@ -244,6 +245,10 @@ class PatroniController(AbstractController):
             self.recursive_update(config, custom_config)
 
         self.recursive_update(config, {
+            'log': {
+                'format': '%(asctime)s %(levelname)s [%(pathname)s:%(lineno)d - %(funcName)s]: %(message)s',
+                'loggers': {'patroni.postgresql.callback_executor': 'DEBUG'}
+            },
             'bootstrap': {
                 'dcs': {
                     'loop_wait': 2,
@@ -251,10 +256,18 @@ class PatroniController(AbstractController):
                         'parameters': {
                             'wal_keep_segments': 100,
                             'archive_mode': 'on',
-                            'archive_command': (PatroniPoolController.ARCHIVE_RESTORE_SCRIPT
+                            'archive_command':
-                                                + ' --mode archive '
+                                (PatroniPoolController.ARCHIVE_RESTORE_SCRIPT
-                                                + '--dirname {} --filename %f --pathname %p').format(
+                                 + ' --mode archive '
-                                                    os.path.join(self._work_directory, 'data', 'wal_archive'))
+                                 + '--dirname {} --filename %f --pathname %p').format(
+                                     os.path.join(self._work_directory, 'data',
+                                                  f'wal_archive{str(self._citus_group or "")}')).replace('\\', '/'),
+                            'restore_command':
+                                (PatroniPoolController.ARCHIVE_RESTORE_SCRIPT
+                                 + ' --mode restore '
+                                 + '--dirname {} --filename %f --pathname %p').format(
+                                     os.path.join(self._work_directory, 'data',
+                                                  f'wal_archive{str(self._citus_group or "")}')).replace('\\', '/')
                         }
                     }
                 }
@@ -649,9 +662,10 @@ class KubernetesController(AbstractExternalDcsController):
             try:
                 if group is not None:
                     scope = '{0}-{1}'.format(scope, group)
-                ep = scope + {'leader': '', 'history': '-config', 'initialize': '-config'}.get(key, '-' + key)
+                rkey = 'leader' if key in ('status', 'failsafe') else key
+                ep = scope + {'leader': '', 'history': '-config', 'initialize': '-config'}.get(rkey, '-' + rkey)
                 e = self._api.read_namespaced_endpoints(ep, self._namespace)
-                if key != 'sync':
+                if key not in ('sync', 'status', 'failsafe'):
                     return e.metadata.annotations[key]
                 else:
                     return json.dumps(e.metadata.annotations)
@@ -687,7 +701,7 @@ class ZooKeeperController(AbstractExternalDcsController):
         self._client = kazoo.client.KazooClient()
 
     def process_name(self):
-        return "zookeeper"
+        return "java .*zookeeper"
 
     def query(self, key, scope='batman', group=None):
         import kazoo.exceptions
@@ -886,22 +900,28 @@ class PatroniPoolController(object):
         }
         self.start(to_name, custom_config=custom_config)
 
+    def backup_restore_config(self, params=None):
+        return {
+            'command': (self.BACKUP_RESTORE_SCRIPT
+                        + ' --sourcedir=' + os.path.join(self.patroni_path, 'data', 'basebackup')).replace('\\', '/'),
+            'test-argument': 'test-value',  # test config mapping approach on custom bootstrap/replica creation
+            **(params or {}),
+        }
+
     def bootstrap_from_backup(self, name, cluster_name):
         custom_config = {
             'scope': cluster_name,
             'bootstrap': {
                 'method': 'backup_restore',
-                'backup_restore': {
+                'backup_restore': self.backup_restore_config({
-                    'command': (self.BACKUP_RESTORE_SCRIPT + ' --sourcedir='
-                                + os.path.join(self.patroni_path, 'data', 'basebackup').replace('\\', '/')),
                     'recovery_conf': {
                         'recovery_target_action': 'promote',
                         'recovery_target_timeline': 'latest',
                         'restore_command': (self.ARCHIVE_RESTORE_SCRIPT + ' --mode restore '
                                             + '--dirname {} --filename %f --pathname %p').format(
                             os.path.join(self.patroni_path, 'data', 'wal_archive_clone').replace('\\', '/'))
-                    }
+                    },
-                }
+                })
             },
             'postgresql': {
                 'authentication': {
@@ -916,17 +936,8 @@ class PatroniPoolController(object):
         custom_config = {
             'scope': cluster_name,
             'postgresql': {
-                'recovery_conf': {
-                    'restore_command': (self.ARCHIVE_RESTORE_SCRIPT + ' --mode restore '
-                                        + '--dirname {} --filename %f --pathname %p')
-                    .format(os.path.join(self.patroni_path, 'data', 'wal_archive').replace('\\', '/'))
-                },
                 'create_replica_methods': ['no_leader_bootstrap'],
-                'no_leader_bootstrap': {
+                'no_leader_bootstrap': self.backup_restore_config({'no_leader': '1'})
-                    'command': (self.BACKUP_RESTORE_SCRIPT + ' --sourcedir='
-                                + os.path.join(self.patroni_path, 'data', 'basebackup').replace('\\', '/')),
-                    'no_leader': '1'
-                }
             }
         }
         self.start(name, custom_config=custom_config)
@@ -1065,6 +1076,8 @@ def before_all(context):
     context.keyfile = os.path.join(context.pctl.output_dir, 'patroni.key')
     context.certfile = os.path.join(context.pctl.output_dir, 'patroni.crt')
     try:
+        if sys.platform == 'darwin' and 'GITHUB_ACTIONS' in os.environ:
+            raise Exception
         with open(os.devnull, 'w') as null:
             ret = subprocess.call(['openssl', 'req', '-nodes', '-new', '-x509', '-subj', '/CN=batman.patroni',
                                    '-addext', 'subjectAltName=IP:127.0.0.1', '-keyout', context.keyfile,
@@ -1082,7 +1095,9 @@ def before_all(context):
                            'PATRONI_RESTAPI_CERTFILE': context.certfile,
                            'PATRONI_RESTAPI_KEYFILE': context.keyfile,
                            'PATRONI_RESTAPI_VERIFY_CLIENT': 'required',
-                           'PATRONI_CTL_INSECURE': 'on'})
+                           'PATRONI_CTL_INSECURE': 'on',
+                           'PATRONI_CTL_CERTFILE': context.certfile,
+                           'PATRONI_CTL_KEYFILE': context.keyfile})
         ctl.update({'cacert': context.certfile, 'certfile': context.certfile, 'keyfile': context.keyfile})
     context.request_executor = PatroniRequest({'ctl': ctl}, True)
     context.dcs_ctl = context.pctl.known_dcs[context.pctl.dcs](context)
@@ -1144,3 +1159,5 @@ def before_scenario(context, scenario):
                 break
     if 'dcs-failsafe' in scenario.effective_tags and not context.dcs_ctl._handle:
         scenario.skip('it is not possible to control state of {0} from tests'.format(context.dcs_ctl.name()))
+    if 'reject-duplicate-name' in scenario.effective_tags and context.dcs_ctl.name() == 'raft':
+        scenario.skip('Flaky test with Raft')

features/ignored_slots.feature

@@ -25,10 +25,10 @@ Feature: ignored slots
     # but Patroni can actually end up dropping them almost immediately, so it's helpful
     # to verify they exist before we begin testing whether they persist through failover
     # cycles.
-    Then postgres1 has a logical replication slot named unmanaged_slot_0 with the test_decoding plugin
+    Then postgres1 has a logical replication slot named unmanaged_slot_0 with the test_decoding plugin after 2 seconds
-    And postgres1 has a logical replication slot named unmanaged_slot_1 with the test_decoding plugin
+    And postgres1 has a logical replication slot named unmanaged_slot_1 with the test_decoding plugin after 2 seconds
-    And postgres1 has a logical replication slot named unmanaged_slot_2 with the test_decoding plugin
+    And postgres1 has a logical replication slot named unmanaged_slot_2 with the test_decoding plugin after 2 seconds
-    And postgres1 has a logical replication slot named unmanaged_slot_3 with the test_decoding plugin
+    And postgres1 has a logical replication slot named unmanaged_slot_3 with the test_decoding plugin after 2 seconds
 
     When I start postgres0
     Then "members/postgres0" key in DCS has role=replica after 10 seconds
@@ -46,16 +46,16 @@ Feature: ignored slots
     And "members/postgres1" key in DCS has role=replica after 10 seconds
     # give Patroni time to sync replication slots
     And I sleep for 2 seconds
-    And postgres1 has a logical replication slot named unmanaged_slot_0 with the test_decoding plugin
+    And postgres1 has a logical replication slot named unmanaged_slot_0 with the test_decoding plugin after 2 seconds
-    And postgres1 has a logical replication slot named unmanaged_slot_1 with the test_decoding plugin
+    And postgres1 has a logical replication slot named unmanaged_slot_1 with the test_decoding plugin after 2 seconds
-    And postgres1 has a logical replication slot named unmanaged_slot_2 with the test_decoding plugin
+    And postgres1 has a logical replication slot named unmanaged_slot_2 with the test_decoding plugin after 2 seconds
-    And postgres1 has a logical replication slot named unmanaged_slot_3 with the test_decoding plugin
+    And postgres1 has a logical replication slot named unmanaged_slot_3 with the test_decoding plugin after 2 seconds
-    And postgres1 does not have a logical replication slot named dummy_slot
+    And postgres1 does not have a replication slot named dummy_slot
 
     # 3. After a failover the server (now a primary) still has the slot.
     When I shut down postgres0
     Then "members/postgres1" key in DCS has role=master after 10 seconds
-    And postgres1 has a logical replication slot named unmanaged_slot_0 with the test_decoding plugin
+    And postgres1 has a logical replication slot named unmanaged_slot_0 with the test_decoding plugin after 2 seconds
-    And postgres1 has a logical replication slot named unmanaged_slot_1 with the test_decoding plugin
+    And postgres1 has a logical replication slot named unmanaged_slot_1 with the test_decoding plugin after 2 seconds
-    And postgres1 has a logical replication slot named unmanaged_slot_2 with the test_decoding plugin
+    And postgres1 has a logical replication slot named unmanaged_slot_2 with the test_decoding plugin after 2 seconds
-    And postgres1 has a logical replication slot named unmanaged_slot_3 with the test_decoding plugin
+    And postgres1 has a logical replication slot named unmanaged_slot_3 with the test_decoding plugin after 2 seconds

features/nostream_node.feature

@@ -0,0 +1,18 @@
+Feature: nostream node
+
+Scenario: check nostream node is recovering from archive
+    When I start postgres0
+    And I configure and start postgres1 with a tag nostream true
+    Then "members/postgres1" key in DCS has replication_state=in archive recovery after 10 seconds
+    And replication works from postgres0 to postgres1 after 30 seconds
+
+@slot-advance
+Scenario: check permanent logical replication slots are not copied
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"wal_level": "logical"}}, "slots":{"test_logical":{"type":"logical","database":"postgres","plugin":"test_decoding"}}}
+    Then I receive a response code 200
+    When I run patronictl.py restart batman postgres0 --force
+    Then postgres0 has a logical replication slot named test_logical with the test_decoding plugin after 10 seconds
+    When I configure and start postgres2 with a tag replicatefrom postgres1
+    Then "members/postgres2" key in DCS has replication_state=streaming after 10 seconds 
+    And postgres1 does not have a replication slot named test_logical
+    And postgres2 does not have a replication slot named test_logical

features/patroni_api.feature

@@ -68,6 +68,7 @@ Scenario: check API requests for the primary-replica pair in the pause mode
 	When I kill postmaster on postgres1
 	And I issue a GET request to http://127.0.0.1:8009/replica
 	Then I receive a response code 503
+	And "members/postgres1" key in DCS has state=stopped after 10 seconds
 	When I run patronictl.py restart batman postgres1 --force
 	Then I receive a response returncode 0
 	Then replication works from postgres0 to postgres1 after 20 seconds
@@ -76,7 +77,7 @@ Scenario: check API requests for the primary-replica pair in the pause mode
 	Then I receive a response code 200
 	And I receive a response state running
 	And I receive a response role replica
-	When I run patronictl.py reinit batman postgres1 --force
+	When I run patronictl.py reinit batman postgres1 --force --wait
 	Then I receive a response returncode 0
 	And I receive a response output "Success: reinitialize for member postgres1"
 	And postgres1 role is the secondary after 30 seconds

features/permanent_slots.feature

@@ -0,0 +1,75 @@
+Feature: permanent slots
+  Scenario: check that physical permanent slots are created
+    Given I start postgres0
+    Then postgres0 is a leader after 10 seconds
+    And there is a non empty initialize key in DCS after 15 seconds
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"slots":{"test_physical":0,"postgres0":0,"postgres1":0,"postgres3":0},"postgresql":{"parameters":{"wal_level":"logical"}}}
+    Then I receive a response code 200
+    And Response on GET http://127.0.0.1:8008/config contains slots after 10 seconds
+    When I start postgres1
+    And I start postgres2
+    And I configure and start postgres3 with a tag replicatefrom postgres2
+    Then postgres0 has a physical replication slot named test_physical after 10 seconds
+    And postgres0 has a physical replication slot named postgres1 after 10 seconds
+    And postgres0 has a physical replication slot named postgres2 after 10 seconds
+    And postgres2 has a physical replication slot named postgres3 after 10 seconds
+
+  @slot-advance
+  Scenario: check that logical permanent slots are created
+    Given I run patronictl.py restart batman postgres0 --force
+    And I issue a PATCH request to http://127.0.0.1:8008/config with {"slots":{"test_logical":{"type":"logical","database":"postgres","plugin":"test_decoding"}}}
+    Then postgres0 has a logical replication slot named test_logical with the test_decoding plugin after 10 seconds
+
+  @slot-advance
+  Scenario: check that permanent slots are created on replicas
+    Given postgres1 has a logical replication slot named test_logical with the test_decoding plugin after 10 seconds
+    Then Logical slot test_logical is in sync between postgres0 and postgres1 after 10 seconds
+    And Logical slot test_logical is in sync between postgres0 and postgres2 after 10 seconds
+    And Logical slot test_logical is in sync between postgres0 and postgres3 after 10 seconds
+    And postgres1 has a physical replication slot named test_physical after 2 seconds
+    And postgres2 has a physical replication slot named test_physical after 2 seconds
+    And postgres3 has a physical replication slot named test_physical after 2 seconds
+
+  @slot-advance
+  Scenario: check permanent physical slots that match with member names
+    Given postgres0 has a physical replication slot named postgres3 after 2 seconds
+    And postgres1 has a physical replication slot named postgres0 after 2 seconds
+    And postgres1 has a physical replication slot named postgres3 after 2 seconds
+    And postgres2 has a physical replication slot named postgres0 after 2 seconds
+    And postgres2 has a physical replication slot named postgres3 after 2 seconds
+    And postgres2 has a physical replication slot named postgres1 after 2 seconds
+    And postgres1 does not have a replication slot named postgres2
+    And postgres3 does not have a replication slot named postgres2
+
+  @slot-advance
+  Scenario: check that permanent slots are advanced on replicas
+    Given I add the table replicate_me to postgres0
+    When I get all changes from logical slot test_logical on postgres0
+    And I get all changes from physical slot test_physical on postgres0
+    Then Logical slot test_logical is in sync between postgres0 and postgres1 after 10 seconds
+    And Physical slot test_physical is in sync between postgres0 and postgres1 after 10 seconds
+    And Logical slot test_logical is in sync between postgres0 and postgres2 after 10 seconds
+    And Physical slot test_physical is in sync between postgres0 and postgres2 after 10 seconds
+    And Logical slot test_logical is in sync between postgres0 and postgres3 after 10 seconds
+    And Physical slot test_physical is in sync between postgres0 and postgres3 after 10 seconds
+    And Physical slot postgres1 is in sync between postgres0 and postgres2 after 10 seconds
+    And Physical slot postgres3 is in sync between postgres2 and postgres0 after 20 seconds
+    And Physical slot postgres3 is in sync between postgres2 and postgres1 after 10 seconds
+    And postgres1 does not have a replication slot named postgres2
+    And postgres3 does not have a replication slot named postgres2
+
+  @slot-advance
+  Scenario: check that only permanent slots are written to the /status key
+    Given "status" key in DCS has test_physical in slots
+    And "status" key in DCS has postgres0 in slots
+    And "status" key in DCS has postgres1 in slots
+    And "status" key in DCS does not have postgres2 in slots
+    And "status" key in DCS has postgres3 in slots
+
+  Scenario: check permanent physical replication slot after failover
+    Given I shut down postgres3
+    And I shut down postgres2
+    And I shut down postgres0
+    Then postgres1 has a physical replication slot named test_physical after 10 seconds
+    And postgres1 has a physical replication slot named postgres0 after 10 seconds
+    And postgres1 has a physical replication slot named postgres3 after 10 seconds

features/priority_failover.feature

@@ -0,0 +1,39 @@
+Feature: priority replication
+  We should check that we can give nodes priority during failover
+
+  Scenario: check failover priority 0 prevents leaderships
+    Given I configure and start postgres0 with a tag failover_priority 1
+    And I configure and start postgres1 with a tag failover_priority 0
+    Then replication works from postgres0 to postgres1 after 20 seconds
+    When I shut down postgres0
+    And there is one of ["following a different leader because I am not allowed to promote"] INFO in the postgres1 patroni log after 5 seconds
+    Then postgres1 role is the secondary after 10 seconds
+    When I start postgres0
+    Then postgres0 role is the primary after 10 seconds
+
+  Scenario: check higher failover priority is respected
+    Given I configure and start postgres2 with a tag failover_priority 1
+    And I configure and start postgres3 with a tag failover_priority 2
+    Then replication works from postgres0 to postgres2 after 20 seconds
+    And replication works from postgres0 to postgres3 after 20 seconds
+    When I shut down postgres0
+    Then postgres3 role is the primary after 10 seconds
+    And there is one of ["postgres3 has equally tolerable WAL position and priority 2, while this node has priority 1","Wal position of postgres3 is ahead of my wal position"] INFO in the postgres2 patroni log after 5 seconds
+
+ Scenario: check conflicting configuration handling
+    When I set nofailover tag in postgres2 config
+    And I issue an empty POST request to http://127.0.0.1:8010/reload
+    Then I receive a response code 202
+    And there is one of ["Conflicting configuration between nofailover: True and failover_priority: 1. Defaulting to nofailover: True"] WARNING in the postgres2 patroni log after 5 seconds
+    And "members/postgres2" key in DCS has tags={'failover_priority': '1', 'nofailover': True} after 10 seconds
+    When I issue a POST request to http://127.0.0.1:8010/failover with {"candidate": "postgres2"}
+    Then I receive a response code 412
+    And I receive a response text "failover is not possible: no good candidates have been found"
+    When I reset nofailover tag in postgres1 config
+    And I issue an empty POST request to http://127.0.0.1:8009/reload
+    Then I receive a response code 202
+    And there is one of ["Conflicting configuration between nofailover: False and failover_priority: 0. Defaulting to nofailover: False"] WARNING in the postgres1 patroni log after 5 seconds
+    And "members/postgres1" key in DCS has tags={'failover_priority': '0', 'nofailover': False} after 10 seconds
+    And I issue a POST request to http://127.0.0.1:8009/failover with {"candidate": "postgres1"}
+    Then I receive a response code 200
+    And postgres1 role is the primary after 10 seconds

features/quorum_commit.feature

@@ -0,0 +1,68 @@
+Feature: quorum commit
+  Check basic workfrlows when quorum commit is enabled
+
+  Scenario: check enable quorum commit and that the only leader promotes after restart
+    Given I start postgres0
+    Then postgres0 is a leader after 10 seconds
+    And there is a non empty initialize key in DCS after 15 seconds
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "synchronous_mode": "quorum"}
+    Then I receive a response code 200
+    And sync key in DCS has leader=postgres0 after 20 seconds
+    And sync key in DCS has quorum=0 after 2 seconds
+    And synchronous_standby_names on postgres0 is set to "_empty_str_" after 2 seconds
+    When I shut down postgres0
+    And sync key in DCS has leader=postgres0 after 2 seconds
+    When I start postgres0
+    Then postgres0 role is the primary after 10 seconds
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"synchronous_mode_strict": true}
+    Then synchronous_standby_names on postgres0 is set to "ANY 1 (*)" after 10 seconds
+
+  Scenario: check failover with one quorum standby
+    Given I start postgres1
+    Then sync key in DCS has sync_standby=postgres1 after 10 seconds
+    And synchronous_standby_names on postgres0 is set to "ANY 1 (postgres1)" after 2 seconds
+    When I shut down postgres0
+    Then postgres1 role is the primary after 10 seconds
+    And sync key in DCS has quorum=0 after 10 seconds
+    Then synchronous_standby_names on postgres1 is set to "ANY 1 (*)" after 10 seconds
+    When I start postgres0
+    Then sync key in DCS has leader=postgres1 after 10 seconds
+    Then sync key in DCS has sync_standby=postgres0 after 10 seconds
+    And synchronous_standby_names on postgres1 is set to "ANY 1 (postgres0)" after 2 seconds
+
+  Scenario: check behavior with three nodes and different replication factor
+    Given I start postgres2
+    Then sync key in DCS has sync_standby=postgres0,postgres2 after 10 seconds
+    And sync key in DCS has quorum=1 after 2 seconds
+    And synchronous_standby_names on postgres1 is set to "ANY 1 (postgres0,postgres2)" after 2 seconds
+    When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_node_count": 2}
+    Then sync key in DCS has quorum=0 after 10 seconds
+    And synchronous_standby_names on postgres1 is set to "ANY 2 (postgres0,postgres2)" after 2 seconds
+
+  Scenario: switch from quorum replication to good old multisync and back
+    Given I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": true, "synchronous_node_count": 1}
+    And I shut down postgres0
+    Then synchronous_standby_names on postgres1 is set to "postgres2" after 10 seconds
+    And sync key in DCS has sync_standby=postgres2 after 10 seconds
+    Then sync key in DCS has quorum=0 after 2 seconds
+    When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": "quorum"}
+    And I start postgres0
+    Then synchronous_standby_names on postgres1 is set to "ANY 1 (postgres0,postgres2)" after 10 seconds
+    And sync key in DCS has sync_standby=postgres0,postgres2 after 10 seconds
+    Then sync key in DCS has quorum=1 after 2 seconds
+
+  Scenario: REST API and patronictl
+    Given I run patronictl.py list batman
+    Then I receive a response returncode 0
+    And I receive a response output "Quorum Standby"
+    And Status code on GET http://127.0.0.1:8008/quorum is 200 after 3 seconds
+    And Status code on GET http://127.0.0.1:8010/quorum is 200 after 3 seconds
+
+  Scenario: nosync node is removed from voters and synchronous_standby_names
+    Given I add tag nosync true to postgres2 config
+    When I issue an empty POST request to http://127.0.0.1:8010/reload
+    Then I receive a response code 202
+    And sync key in DCS has quorum=0 after 10 seconds
+    And sync key in DCS has sync_standby=postgres0 after 10 seconds
+    And synchronous_standby_names on postgres1 is set to "ANY 1 (postgres0)" after 2 seconds
+    And Status code on GET http://127.0.0.1:8010/quorum is 503 after 10 seconds

features/recovery.feature

@@ -14,6 +14,8 @@ Feature: recovery
     Then I receive a response code 200
     And I receive a response role master
     And I receive a response timeline 1
+    And "members/postgres0" key in DCS has state=running after 12 seconds
+    And replication works from postgres0 to postgres1 after 15 seconds
 
   Scenario: check immediate failover when master_start_timeout=0
     Given I issue a PATCH request to http://127.0.0.1:8008/config with {"master_start_timeout": 0}

features/standby_cluster.feature

@@ -22,14 +22,11 @@ Feature: standby cluster
   Scenario: check permanent logical slots are synced to the replica
     Given I run patronictl.py restart batman postgres1 --force
     Then Logical slot test_logical is in sync between postgres0 and postgres1 after 10 seconds
-    When I add the table replicate_me to postgres1
-    And I get all changes from logical slot test_logical on postgres1
-    Then Logical slot test_logical is in sync between postgres0 and postgres1 after 10 seconds
 
   Scenario: Detach exiting node from the cluster
     When I shut down postgres1
     Then postgres0 is a leader after 10 seconds
-    And "members/postgres0" key in DCS has role=master after 3 seconds
+    And "members/postgres0" key in DCS has role=master after 5 seconds
     When I issue a GET request to http://127.0.0.1:8008/
     Then I receive a response code 200
 
@@ -50,21 +47,26 @@ Feature: standby cluster
     And there is a postgres1_cb.log with "on_role_change standby_leader batman1" in postgres1 data directory
     When I start postgres2 in a cluster batman1
     Then postgres2 role is the replica after 24 seconds
+    And postgres2 is replicating from postgres1 after 10 seconds
     And table foo is present on postgres2 after 20 seconds
     When I issue a GET request to http://127.0.0.1:8010/patroni
     Then I receive a response code 200
     And I receive a response replication_state streaming
-    And postgres1 does not have a logical replication slot named test_logical
+    And postgres1 does not have a replication slot named test_logical
+
+  Scenario: check switchover
+    Given I run patronictl.py switchover batman1 --force
+    Then Status code on GET http://127.0.0.1:8010/standby_leader is 200 after 10 seconds
+    And postgres1 is replicating from postgres2 after 32 seconds
+    And there is a postgres2_cb.log with "on_start replica batman1\non_role_change standby_leader batman1" in postgres2 data directory
 
   Scenario: check failover
-    When I kill postgres1
+    When I kill postgres2
-    And I kill postmaster on postgres1
+    And I kill postmaster on postgres2
-    Then postgres2 is replicating from postgres0 after 32 seconds
+    Then postgres1 is replicating from postgres0 after 32 seconds
-    When I issue a GET request to http://127.0.0.1:8010/primary
+    And Status code on GET http://127.0.0.1:8009/standby_leader is 200 after 10 seconds
+    When I issue a GET request to http://127.0.0.1:8009/primary
     Then I receive a response code 503
-    And I sleep for 3 seconds
-    When I issue a GET request to http://127.0.0.1:8010/standby_leader
-    Then I receive a response code 200
     And I receive a response role standby_leader
-    And replication works from postgres0 to postgres2 after 15 seconds
+    And replication works from postgres0 to postgres1 after 15 seconds
-    And there is a postgres2_cb.log with "on_start replica batman1\non_role_change standby_leader batman1" in postgres2 data directory
+    And there is a postgres1_cb.log with "on_role_change replica batman1\non_role_change standby_leader batman1" in postgres1 data directory

features/steps/basic_replication.py

@@ -1,3 +1,4 @@
+import json
 import patroni.psycopg as pg
 
 from behave import step, then
@@ -21,7 +22,7 @@ def start_duplicate_patroni(context, name, port):
         context.pctl.start('dup-' + name, custom_config=config)
         assert False, "Process was expected to fail"
     except AssertionError as e:
-        assert 'is not running after being started' in str(e),\
+        assert 'is not running after being started' in str(e), \
             "No error was raised by duplicate start of {0} ".format(name)
 
 
@@ -35,16 +36,27 @@ def kill_patroni(context, name):
     return context.pctl.stop(name, kill=True)
 
 
-@step('I kill postmaster on {name:w}')
+@step('I shut down postmaster on {name:w}')
 def stop_postgres(context, name):
     return context.pctl.stop(name, postgres=True)
 
 
+@step('I kill postmaster on {name:w}')
+def kill_postgres(context, name):
+    return context.pctl.stop(name, kill=True, postgres=True)
+
+
+def get_wal_name(context, pg_name):
+    version = context.pctl.query(pg_name, "SHOW server_version_num").fetchone()[0]
+    return 'xlog' if int(version) / 10000 < 10 else 'wal'
+
+
 @step('I add the table {table_name:w} to {pg_name:w}')
 def add_table(context, table_name, pg_name):
     # parse the configuration file and get the port
     try:
         context.pctl.query(pg_name, "CREATE TABLE public.{0}()".format(table_name))
+        context.pctl.query(pg_name, "SELECT pg_switch_{0}()".format(get_wal_name(context, pg_name)))
     except pg.Error as e:
         assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e)
 
@@ -53,9 +65,7 @@ def add_table(context, table_name, pg_name):
 def toggle_wal_replay(context, action, pg_name):
     # pause or resume the wal replay process
     try:
-        version = context.pctl.query(pg_name, "SHOW server_version_num").fetchone()[0]
+        context.pctl.query(pg_name, "SELECT pg_{0}_replay_{1}()".format(get_wal_name(context, pg_name), action))
-        wal_name = 'xlog' if int(version) / 10000 < 10 else 'wal'
-        context.pctl.query(pg_name, "SELECT pg_{0}_replay_{1}()".format(wal_name, action))
     except pg.Error as e:
         assert False, "Error during {0} wal recovery on {1}: {2}".format(action, pg_name, e)
 
@@ -88,14 +98,14 @@ def table_is_present_on(context, table_name, pg_name, max_replication_delay):
             break
         sleep(1)
     else:
-        assert False,\
+        assert False, \
             "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay)
 
 
 @then('{pg_name:w} role is the {pg_role:w} after {max_promotion_timeout:d} seconds')
 def check_role(context, pg_name, pg_role, max_promotion_timeout):
     max_promotion_timeout *= context.timeout_multiplier
-    assert context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)),\
+    assert context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)), \
         "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
 
 
@@ -105,11 +115,18 @@ def replication_works(context, primary, replica, time_limit):
     context.execute_steps(u"""
         When I add the table test_{0} to {1}
         Then table test_{0} is present on {2} after {3} seconds
-    """.format(int(time()), primary, replica, time_limit))
+    """.format(str(time()).replace('.', '_').replace(',', '_'), primary, replica, time_limit))
 
 
-@then('there is a "{message}" {level:w} in the {node} patroni log')
+@step('there is one of {message_list} {level:w} in the {node} patroni log after {timeout:d} seconds')
-def check_patroni_log(context, message, level, node):
+def check_patroni_log(context, message_list, level, node, timeout):
-    messsages_of_level = context.pctl.read_patroni_log(node, level)
+    timeout *= context.timeout_multiplier
-    assert any(message in line for line in messsages_of_level),\
+    message_list = json.loads(message_list)
-        "There was no {0} {1} in the {2} patroni log".format(message, level, node)
+
+    for _ in range(int(timeout)):
+        messsages_of_level = context.pctl.read_patroni_log(node, level)
+        if any(any(message in line for line in messsages_of_level) for message in message_list):
+            break
+        sleep(1)
+    else:
+        assert False, f"There were none of {message_list} {level} in the {node} patroni log after {timeout} seconds"

features/steps/cascading_replication.py

@@ -28,7 +28,7 @@ def check_member(context, name, key, value, time_limit):
     while time.time() < max_time:
         try:
             response = json.loads(context.dcs_ctl.query(name))
-            dcs_value = response.get(key)
+            dcs_value = str(response.get(key))
             if dcs_value == value:
                 return
         except Exception:

features/steps/citus.py

@@ -115,15 +115,21 @@ def count_rows(context, name):
     assert rows == context.insert_counter, "Distributed table doesn't have expected amount of rows"
 
 
-@step("There is a transaction in progress on {name:w} changing pg_dist_node")
+@step("there is a transaction in progress on {name:w} changing pg_dist_node after {time_limit:d} seconds")
-def check_transaction(context, name):
+def check_transaction(context, name, time_limit):
-    cur = context.pctl.query(name, "SELECT xact_start FROM pg_stat_activity WHERE pid <> pg_backend_pid()"
+    time_limit *= context.timeout_multiplier
-                                   " AND state = 'idle in transaction' AND query ~ 'citus_update_node'")
+    max_time = time.time() + int(time_limit)
-    assert cur.rowcount == 1, "There is no idle in transaction updating pg_dist_node"
+    while time.time() < max_time:
-    context.xact_start = cur.fetchone()[0]
+        cur = context.pctl.query(name, "SELECT xact_start FROM pg_stat_activity WHERE pid <> pg_backend_pid()"
+                                       " AND state = 'idle in transaction' AND query ~ 'citus_update_node'")
+        if cur.rowcount == 1:
+            context.xact_start = cur.fetchone()[0]
+            return
+        time.sleep(1)
+    assert False, f"There is no idle in transaction on {name} updating pg_dist_node after {time_limit} seconds"
 
 
 @step("a transaction finishes in {timeout:d} seconds")
 def check_transaction_timeout(context, timeout):
-    assert (datetime.now(tzutc) - context.xact_start).seconds > timeout,\
+    assert (datetime.now(tzutc) - context.xact_start).seconds >= timeout, \
         "a transaction finished earlier than in {0} seconds".format(timeout)

features/steps/patroni_api.py

@@ -98,7 +98,7 @@ def do_run(context, cmd):
 @then('I receive a response {component:w} {data}')
 def check_response(context, component, data):
     if component == 'code':
-        assert context.status_code == int(data),\
+        assert context.status_code == int(data), \
             "status code {0} != {1}, response: {2}".format(context.status_code, data, context.response)
     elif component == 'returncode':
         assert context.status_code == int(data), "return code {0} != {1}, {2}".format(context.status_code,
@@ -128,6 +128,12 @@ def scheduled_restart(context, url, in_seconds, data):
     context.execute_steps(u"""Given I issue a POST request to {0}/restart with {1}""".format(url, json.dumps(data)))
 
 
+@step('I {action:w} {tag:w} tag in {pg_name:w} config')
+def add_bool_tag_to_config(context, action, tag, pg_name):
+    value = action == 'set'
+    context.pctl.add_tag_to_config(pg_name, tag, value)
+
+
 @step('I add tag {tag:w} {value:w} to {pg_name:w} config')
 def add_tag_to_config(context, tag, value, pg_name):
     context.pctl.add_tag_to_config(pg_name, tag, value)
@@ -158,7 +164,7 @@ def check_http_response(context, url, value, timeout, negate=False):
             break
         time.sleep(1)
     else:
-        assert False,\
+        assert False, \
             "Value {0} is {1} present in response after {2} seconds".format(value, "not" if not negate else "", timeout)
 
 

features/steps/quorum_commit.py

@@ -0,0 +1,60 @@
+import json
+import re
+import time
+
+from behave import step, then
+
+
+@step('sync key in DCS has {key:w}={value} after {time_limit:d} seconds')
+def check_sync(context, key, value, time_limit):
+    time_limit *= context.timeout_multiplier
+    max_time = time.time() + int(time_limit)
+    dcs_value = None
+    while time.time() < max_time:
+        try:
+            response = json.loads(context.dcs_ctl.query('sync'))
+            dcs_value = response.get(key)
+            if key == 'sync_standby' and set((dcs_value or '').split(',')) == set(value.split(',')):
+                return
+            elif str(dcs_value) == value:
+                return
+        except Exception:
+            pass
+        time.sleep(1)
+    assert False, "sync does not have {0}={1} (found {2}) in dcs after {3} seconds".format(key, value,
+                                                                                           dcs_value, time_limit)
+
+
+def _parse_synchronous_standby_names(value):
+    if '(' in value:
+        m = re.match(r'.*(\d+) \(([^)]+)\)', value)
+        expected_value = set(m.group(2).split())
+        expected_num = m.group(1)
+    else:
+        expected_value = set([value])
+        expected_num = '1'
+    return expected_num, expected_value
+
+
+@then('synchronous_standby_names on {name:2} is set to "{value}" after {time_limit:d} seconds')
+def check_synchronous_standby_names(context, name, value, time_limit):
+    time_limit *= context.timeout_multiplier
+    max_time = time.time() + int(time_limit)
+
+    if value == '_empty_str_':
+        value = ''
+
+    expected_num, expected_value = _parse_synchronous_standby_names(value)
+
+    ssn = None
+    while time.time() < max_time:
+        try:
+            ssn = context.pctl.query(name, "SHOW synchronous_standby_names").fetchone()[0]
+            db_num, db_value = _parse_synchronous_standby_names(ssn)
+            if expected_value == db_value and expected_num == db_num:
+                return
+        except Exception:
+            pass
+        time.sleep(1)
+    assert False, "synchronous_standby_names is not set to '{0}' (found '{1}') after {2} seconds".format(value, ssn,
+                                                                                                         time_limit)

features/steps/slots.py

@@ -1,3 +1,4 @@
+import json
 import time
 
 from behave import step, then
@@ -15,21 +16,30 @@ def create_logical_replication_slot(context, slot_name, pg_name, plugin):
         assert False, "Error creating slot {0} on {1} with plugin {2}".format(slot_name, pg_name, plugin)
 
 
-@then('{pg_name:w} has a logical replication slot named {slot_name} with the {plugin:w} plugin')
+@step('{pg_name:w} has a logical replication slot named {slot_name}'
-def has_logical_replication_slot(context, pg_name, slot_name, plugin):
+      ' with the {plugin:w} plugin after {time_limit:d} seconds')
-    try:
+@then('{pg_name:w} has a logical replication slot named {slot_name}'
-        row = context.pctl.query(pg_name, ("SELECT slot_type, plugin FROM pg_replication_slots"
+      ' with the {plugin:w} plugin after {time_limit:d} seconds')
-                                           " WHERE slot_name = '{0}'").format(slot_name)).fetchone()
+def has_logical_replication_slot(context, pg_name, slot_name, plugin, time_limit):
-        assert row, "Couldn't find replication slot named {0}".format(slot_name)
+    time_limit *= context.timeout_multiplier
-        assert row[0] == "logical", "Found replication slot named {0} but wasn't a logical slot".format(slot_name)
+    max_time = time.time() + int(time_limit)
-        assert row[1] == plugin, ("Found replication slot named {0} but was using plugin "
+    while time.time() < max_time:
-                                  "{1} rather than {2}").format(slot_name, row[1], plugin)
+        try:
-    except pg.Error:
+            row = context.pctl.query(pg_name, ("SELECT slot_type, plugin FROM pg_replication_slots"
-        assert False, "Error looking for slot {0} on {1} with plugin {2}".format(slot_name, pg_name, plugin)
+                                               f" WHERE slot_name = '{slot_name}'")).fetchone()
+            if row:
+                assert row[0] == "logical", f"Replication slot {slot_name} isn't a logical but {row[0]}"
+                assert row[1] == plugin, f"Replication slot {slot_name} using plugin {row[1]} rather than {plugin}"
+                return
+        except Exception:
+            pass
+        time.sleep(1)
+    assert False, f"Error looking for slot {slot_name} on {pg_name} with plugin {plugin}"
 
 
-@then('{pg_name:w} does not have a logical replication slot named {slot_name}')
+@step('{pg_name:w} does not have a replication slot named {slot_name:w}')
-def does_not_have_logical_replication_slot(context, pg_name, slot_name):
+@then('{pg_name:w} does not have a replication slot named {slot_name:w}')
+def does_not_have_replication_slot(context, pg_name, slot_name):
     try:
         row = context.pctl.query(pg_name, ("SELECT 1 FROM pg_replication_slots"
                                            " WHERE slot_name = '{0}'").format(slot_name)).fetchone()
@@ -38,13 +48,14 @@ def does_not_have_logical_replication_slot(context, pg_name, slot_name):
         assert False, "Error looking for slot {0} on {1}".format(slot_name, pg_name)
 
 
-@step('Logical slot {slot_name:w} is in sync between {pg_name1:w} and {pg_name2:w} after {time_limit:d} seconds')
+@step('{slot_type:w} slot {slot_name:w} is in sync between {pg_name1:w} and {pg_name2:w} after {time_limit:d} seconds')
-def logical_slots_in_sync(context, slot_name, pg_name1, pg_name2, time_limit):
+def slots_in_sync(context, slot_type, slot_name, pg_name1, pg_name2, time_limit):
     time_limit *= context.timeout_multiplier
     max_time = time.time() + int(time_limit)
+    column = 'confirmed_flush_lsn' if slot_type.lower() == 'logical' else 'restart_lsn'
+    query = f"SELECT {column} FROM pg_replication_slots WHERE slot_name = '{slot_name}'"
     while time.time() < max_time:
         try:
-            query = "SELECT confirmed_flush_lsn FROM pg_replication_slots WHERE slot_name = '{0}'".format(slot_name)
             slot1 = context.pctl.query(pg_name1, query).fetchone()
             slot2 = context.pctl.query(pg_name2, query).fetchone()
             if slot1[0] == slot2[0]:
@@ -52,9 +63,43 @@ def logical_slots_in_sync(context, slot_name, pg_name1, pg_name2, time_limit):
         except Exception:
             pass
         time.sleep(1)
-    assert False, "Logical slot {0} is not in sync between {1} and {2}".format(slot_name, pg_name1, pg_name2)
+    assert False, \
+        f"{slot_type} slot {slot_name} is not in sync between {pg_name1} and {pg_name2} after {time_limit} seconds"
 
 
 @step('I get all changes from logical slot {slot_name:w} on {pg_name:w}')
 def logical_slot_get_changes(context, slot_name, pg_name):
     context.pctl.query(pg_name, "SELECT * FROM pg_logical_slot_get_changes('{0}', NULL, NULL)".format(slot_name))
+
+
+@step('I get all changes from physical slot {slot_name:w} on {pg_name:w}')
+def physical_slot_get_changes(context, slot_name, pg_name):
+    context.pctl.query(pg_name, f"SELECT * FROM pg_replication_slot_advance('{slot_name}', pg_current_wal_lsn())")
+
+
+@step('{pg_name:w} has a physical replication slot named {slot_name} after {time_limit:d} seconds')
+def has_physical_replication_slot(context, pg_name, slot_name, time_limit):
+    time_limit *= context.timeout_multiplier
+    max_time = time.time() + int(time_limit)
+    query = f"SELECT * FROM pg_catalog.pg_replication_slots WHERE slot_type = 'physical' AND slot_name = '{slot_name}'"
+    while time.time() < max_time:
+        try:
+            row = context.pctl.query(pg_name, query).fetchone()
+            if row:
+                return
+        except Exception:
+            pass
+        time.sleep(1)
+    assert False, f"Physical slot {slot_name} doesn't exist after {time_limit} seconds"
+
+
+@step('"{name}" key in DCS has {subkey:w} in {key:w}')
+def dcs_key_contains(context, name, subkey, key):
+    response = json.loads(context.dcs_ctl.query(name))
+    assert key in response and subkey in response[key], f"{name} key in DCS doesn't have {subkey} in {key}"
+
+
+@step('"{name}" key in DCS does not have {subkey:w} in {key:w}')
+def dcs_key_does_not_contain(context, name, subkey, key):
+    response = json.loads(context.dcs_ctl.query(name))
+    assert key not in response or subkey not in response[key], f"{name} key in DCS has {subkey} in {key}"

features/steps/standby_cluster.py

@@ -15,9 +15,7 @@ def start_patroni(context, name, cluster_name):
         "scope": cluster_name,
         "postgresql": {
             "callbacks": callbacks(context, name),
-            "backup_restore": {
+            "backup_restore": context.pctl.backup_restore_config()
-                "command": (context.pctl.PYTHON + " features/backup_restore.py --sourcedir="
-                            + os.path.join(context.pctl.patroni_path, 'data', 'basebackup').replace('\\', '/'))}
         }
     })
 
@@ -34,6 +32,7 @@ def start_patroni_standby_cluster(context, name, cluster_name, name2):
                 "ttl": 20,
                 "loop_wait": 2,
                 "retry_timeout": 5,
+                "synchronous_mode": True,  # should be completely ignored
                 "standby_cluster": {
                     "host": "localhost",
                     "port": port,

kubernetes/Dockerfile

@@ -1,4 +1,4 @@
-FROM postgres:15
+FROM postgres:16
 LABEL maintainer="Alexander Kukushkin <[email protected]>"
 
 RUN export DEBIAN_FRONTEND=noninteractive \
@@ -9,8 +9,8 @@ RUN export DEBIAN_FRONTEND=noninteractive \
             | xargs apt-get install -y vim-tiny curl jq locales git python3-pip python3-wheel \
     ## Make sure we have a en_US.UTF-8 locale available
     && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
-    && pip3 install setuptools \
+    && pip3 install --break-system-packages setuptools \
-    && pip3 install 'git+https://github.com/zalando/patroni.git#egg=patroni[kubernetes]' \
+    && pip3 install --break-system-packages 'git+https://github.com/zalando/patroni.git#egg=patroni[kubernetes]' \
     && PGHOME=/home/postgres \
     && mkdir -p $PGHOME \
     && chown postgres $PGHOME \

kubernetes/Dockerfile.citus

@@ -1,4 +1,4 @@
-FROM postgres:15
+FROM postgres:16
 LABEL maintainer="Alexander Kukushkin <[email protected]>"
 
 RUN export DEBIAN_FRONTEND=noninteractive \
@@ -10,12 +10,24 @@ RUN export DEBIAN_FRONTEND=noninteractive \
             | xargs apt-get install -y busybox vim-tiny curl jq less locales git python3-pip python3-wheel lsb-release \
     ## Make sure we have a en_US.UTF-8 locale available
     && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
-    && echo "deb [signed-by=/etc/apt/trusted.gpg.d/citusdata_community.gpg] https://packagecloud.io/citusdata/community/debian/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/citusdata_community.list \
+    && if [ $(dpkg --print-architecture) = 'arm64' ]; then \
-    && curl -sL https://packagecloud.io/citusdata/community/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/citusdata_community.gpg \
+        apt-get install -y postgresql-server-dev-16 \
-    && apt-get update -y \
+                           gcc make autoconf \
-    && apt-get -y install postgresql-$PG_MAJOR-citus-11.3 \
+                           libc6-dev flex libcurl4-gnutls-dev \
-    && pip3 install setuptools \
+                           libicu-dev libkrb5-dev liblz4-dev \
-    && pip3 install 'git+https://github.com/zalando/patroni.git#egg=patroni[kubernetes]' \
+                           libpam0g-dev libreadline-dev libselinux1-dev\
+                           libssl-dev libxslt1-dev libzstd-dev uuid-dev \
+        && git clone -b "main" https://github.com/citusdata/citus.git \
+        && MAKEFLAGS="-j $(grep -c ^processor /proc/cpuinfo)" \
+        && cd citus && ./configure && make install && cd ../ && rm -rf /citus; \
+    else \
+        echo "deb [signed-by=/etc/apt/trusted.gpg.d/citusdata_community.gpg] https://packagecloud.io/citusdata/community/debian/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/citusdata_community.list \
+        && curl -sL https://packagecloud.io/citusdata/community/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/citusdata_community.gpg \
+        && apt-get update -y \
+        && apt-get -y install postgresql-16-citus-12.1; \
+    fi \
+    && pip3 install --break-system-packages setuptools \
+    && pip3 install --break-system-packages 'git+https://github.com/zalando/patroni.git#egg=patroni[kubernetes]' \
     && PGHOME=/home/postgres \
     && mkdir -p $PGHOME \
     && chown postgres $PGHOME \
@@ -26,6 +38,9 @@ RUN export DEBIAN_FRONTEND=noninteractive \
     && chmod 664 /etc/passwd \
     # Clean up
     && apt-get remove -y git python3-pip python3-wheel \
+               postgresql-server-dev-16 gcc make autoconf \
+               libc6-dev flex libicu-dev libkrb5-dev liblz4-dev \
+               libpam0g-dev libreadline-dev libselinux1-dev libssl-dev libxslt1-dev libzstd-dev uuid-dev \
     && apt-get autoremove -y \
     && apt-get clean -y \
     && rm -rf /var/lib/apt/lists/* /root/.cache

patroni/__init__.py

@@ -3,23 +3,14 @@
 :var PATRONI_ENV_PREFIX: prefix for Patroni related configuration environment variables.
 :var KUBERNETES_ENV_PREFIX: prefix for Kubernetes related configuration environment variables.
 :var MIN_PSYCOPG2: minimum version of :mod:`psycopg2` required by Patroni to work.
+:var MIN_PSYCOPG3: minimum version of :mod:`psycopg` required by Patroni to work.
 """
-
+from typing import Iterator, Tuple
-import sys
-
-from typing import Any, Callable, Iterator, Tuple
 
 PATRONI_ENV_PREFIX = 'PATRONI_'
 KUBERNETES_ENV_PREFIX = 'KUBERNETES_'
 MIN_PSYCOPG2 = (2, 5, 4)
-
+MIN_PSYCOPG3 = (3, 0, 0)
-
-def fatal(string: str, *args: Any) -> None:
-    """Write a fatal message to stderr and exit with code ``1``.
-
-    :param string: message to be written before exiting.
-    """
-    sys.exit('FATAL: ' + string.format(*args))
 
 
 def parse_version(version: str) -> Tuple[int, ...]:
@@ -28,25 +19,25 @@ def parse_version(version: str) -> Tuple[int, ...]:
     .. note::
         Designed for easy comparison of software versions in Python.
 
-    :param version: human-readable software version, e.g. ``2.5.4``.
+    :param version: human-readable software version, e.g. ``2.5.4.dev1 (dt dec pq3 ext lo64)``.
 
     :returns: tuple of *version* parts, each part as an integer.
 
     :Example:
 
-        >>> parse_version('2.5.4')
+        >>> parse_version('2.5.4.dev1 (dt dec pq3 ext lo64)')
         (2, 5, 4)
     """
     def _parse_version(version: str) -> Iterator[int]:
         """Yield each part of a human-readable version string as an integer.
 
-        :param version: human-readable software version, e.g. ``2.5.4``.
+        :param version: human-readable software version, e.g. ``2.5.4.dev1``.
 
         :yields: each part of *version* as an integer.
 
         :Example:
 
-            >>> tuple(_parse_version('2.5.4'))
+            >>> tuple(_parse_version('2.5.4.dev1'))
             (2, 5, 4)
         """
         for e in version.split('.'):
@@ -55,40 +46,3 @@ def parse_version(version: str) -> Tuple[int, ...]:
             except ValueError:
                 break
     return tuple(_parse_version(version.split(' ')[0]))
-
-
-def check_psycopg(_min_psycopg2: Tuple[int, ...] = MIN_PSYCOPG2,
-                  _parse_version: Callable[[str], Tuple[int, ...]] = parse_version) -> None:
-    """Ensure at least one among :mod:`psycopg2` or :mod:`psycopg` libraries are available in the environment.
-
-    .. note::
-        We pass ``MIN_PSYCOPG2`` and :func:`parse_version` as arguments to simplify usage of :func:`check_psycopg` from
-        the ``setup.py``.
-
-    .. note::
-        Patroni chooses :mod:`psycopg2` over :mod:`psycopg`, if possible.
-
-        If nothing meeting the requirements is found, then exit with a fatal message.
-
-    :param _min_psycopg2: minimum required version in case :mod:`psycopg2` is chosen.
-    :param _parse_version: function used to parse :mod:`psycopg2`/:mod:`psycopg` version into a comparable object.
-    """
-    min_psycopg2_str = '.'.join(map(str, _min_psycopg2))
-
-    # try psycopg2
-    try:
-        from psycopg2 import __version__
-        if _parse_version(__version__) >= _min_psycopg2:
-            return
-        version_str = __version__.split(' ')[0]
-    except ImportError:
-        version_str = None
-
-    # try psycopg3
-    try:
-        from psycopg import __version__
-    except ImportError:
-        error = 'Patroni requires psycopg2>={0}, psycopg2-binary, or psycopg>=3.0'.format(min_psycopg2_str)
-        if version_str is not None:
-            error += ', but only psycopg2=={0} is available'.format(version_str)
-        fatal(error)

patroni/__main__.py

@@ -1,3 +1,8 @@
+"""Patroni main entry point.
+
+Implement ``patroni`` main daemon and expose its entry point.
+"""
+
 import logging
 import os
 import signal
@@ -5,9 +10,11 @@ import sys
 import time
 
 from argparse import Namespace
-from typing import Any, Dict, Optional, TYPE_CHECKING
+from typing import Any, Dict, List, Optional, TYPE_CHECKING
 
+from patroni import MIN_PSYCOPG2, MIN_PSYCOPG3, parse_version
 from patroni.daemon import AbstractPatroniDaemon, abstract_main, get_base_arg_parser
+from patroni.tags import Tags
 
 if TYPE_CHECKING:  # pragma: no cover
     from .config import Config
@@ -15,9 +22,33 @@ if TYPE_CHECKING:  # pragma: no cover
 logger = logging.getLogger(__name__)
 
 
-class Patroni(AbstractPatroniDaemon):
+class Patroni(AbstractPatroniDaemon, Tags):
+    """Implement ``patroni`` command daemon.
+
+    :ivar version: Patroni version.
+    :ivar dcs: DCS object.
+    :ivar watchdog: watchdog handler, if configured to use watchdog.
+    :ivar postgresql: managed Postgres instance.
+    :ivar api: REST API server instance of this node.
+    :ivar request: wrapper for performing HTTP requests.
+    :ivar ha: HA handler.
+    :ivar next_run: time when to run the next HA loop cycle.
+    :ivar scheduled_restart: when a restart has been scheduled to occur, if any. In that case, should contain two keys:
+        * ``schedule``: timestamp when restart should occur;
+        * ``postmaster_start_time``: timestamp when Postgres was last started.
+    """
 
     def __init__(self, config: 'Config') -> None:
+        """Create a :class:`Patroni` instance with the given *config*.
+
+        Get a connection to the DCS, configure watchdog (if required), set up Patroni interface with Postgres, configure
+        the HA loop and bring the REST API up.
+
+        .. note::
+            Expected to be instantiated and run through :func:`~patroni.daemon.abstract_main`.
+
+        :param config: Patroni configuration.
+        """
         from patroni.api import RestApiServer
         from patroni.dcs import get_dcs
         from patroni.ha import Ha
@@ -37,15 +68,26 @@ class Patroni(AbstractPatroniDaemon):
         self.watchdog = Watchdog(self.config)
         self.load_dynamic_configuration()
 
-        self.postgresql = Postgresql(self.config['postgresql'])
+        self.postgresql = Postgresql(self.config['postgresql'], self.dcs.mpp)
         self.api = RestApiServer(self, self.config['restapi'])
         self.ha = Ha(self)
 
-        self.tags = self.get_tags()
+        self._tags = self._get_tags()
         self.next_run = time.time()
         self.scheduled_restart: Dict[str, Any] = {}
 
     def load_dynamic_configuration(self) -> None:
+        """Load Patroni dynamic configuration.
+
+        Load dynamic configuration from the DCS, if `/config` key is available in the DCS, otherwise fall back to
+        ``bootstrap.dcs`` section from the configuration file.
+
+        If the DCS connection fails returning the exception :class:`~patroni.exceptions.DCSError` an attempt will be
+        remade every 5 seconds.
+
+        .. note::
+            This method is called only once, at the time when Patroni is started.
+        """
         from patroni.exceptions import DCSError
         while True:
             try:
@@ -74,29 +116,40 @@ class Patroni(AbstractPatroniDaemon):
         if not isinstance(member, Member):
             return
         try:
-            _ = self.request(member, endpoint="/liveness")
+            # Silence annoying WARNING: Retrying (...) messages when Patroni is quickly restarted.
+            # At this moment we don't have custom log levels configured and hence shouldn't lose anything useful.
+            self.logger.update_loggers({'urllib3.connectionpool': 'ERROR'})
+            _ = self.request(member, endpoint="/liveness", timeout=3)
             logger.fatal("Can't start; there is already a node named '%s' running", self.config['name'])
             sys.exit(1)
         except Exception:
-            return
+            self.logger.update_loggers({})
 
-    def get_tags(self) -> Dict[str, Any]:
+    def _get_tags(self) -> Dict[str, Any]:
-        return {tag: value for tag, value in self.config.get('tags', {}).items()
+        """Get tags configured for this node, if any.
-                if tag not in ('clonefrom', 'nofailover', 'noloadbalance', 'nosync') or value}
 
-    @property
+        :returns: a dictionary of tags set for this node.
-    def nofailover(self) -> bool:
+        """
-        return bool(self.tags.get('nofailover', False))
+        return self._filter_tags(self.config.get('tags', {}))
-
-    @property
-    def nosync(self) -> bool:
-        return bool(self.tags.get('nosync', False))
 
     def reload_config(self, sighup: bool = False, local: Optional[bool] = False) -> None:
+        """Apply new configuration values for ``patroni`` daemon.
+
+        Reload:
+            * Cached tags;
+            * Request wrapper configuration;
+            * REST API configuration;
+            * Watchdog configuration;
+            * Postgres configuration;
+            * DCS configuration.
+
+        :param sighup: if it is related to a SIGHUP signal.
+        :param local: if there has been changes to the local configuration file.
+        """
         try:
             super(Patroni, self).reload_config(sighup, local)
             if local:
-                self.tags = self.get_tags()
+                self._tags = self._get_tags()
                 self.request.reload_config(self.config)
             if local or sighup and self.api.reload_local_certificate():
                 self.api.reload_config(self.config['restapi'])
@@ -107,14 +160,16 @@ class Patroni(AbstractPatroniDaemon):
             logger.exception('Failed to reload config_file=%s', self.config.config_file)
 
     @property
-    def replicatefrom(self):
+    def tags(self) -> Dict[str, Any]:
-        return self.tags.get('replicatefrom')
+        """Tags configured for this node, if any."""
-
+        return self._tags
-    @property
-    def noloadbalance(self):
-        return bool(self.tags.get('noloadbalance', False))
 
     def schedule_next_run(self) -> None:
+        """Schedule the next run of the ``patroni`` daemon main loop.
+
+        Next run is scheduled based on previous run plus value of ``loop_wait`` configuration from DCS. If that has
+        already been exceeded, run the next cycle immediately.
+        """
         self.next_run += self.dcs.loop_wait
         current_time = time.time()
         nap_time = self.next_run - current_time
@@ -128,11 +183,21 @@ class Patroni(AbstractPatroniDaemon):
             self.next_run = time.time()
 
     def run(self) -> None:
+        """Run ``patroni`` daemon process main loop.
+
+        Start the REST API and keep running HA cycles every ``loop_wait`` seconds.
+        """
         self.api.start()
         self.next_run = time.time()
         super(Patroni, self).run()
 
     def _run_cycle(self) -> None:
+        """Run a cycle of the ``patroni`` daemon main loop.
+
+        Run an HA cycle and schedule the next cycle run. If any dynamic configuration change request is detected, apply
+        the change and cache the new dynamic configuration values in ``patroni.dynamic.json`` file under Postgres data
+        directory.
+        """
         logger.info(self.ha.run_cycle())
 
         if self.dcs.cluster and self.dcs.cluster.config and self.dcs.cluster.config.data \
@@ -145,6 +210,10 @@ class Patroni(AbstractPatroniDaemon):
         self.schedule_next_run()
 
     def _shutdown(self) -> None:
+        """Perform shutdown of ``patroni`` daemon process.
+
+        Shut down the REST API and the HA handler.
+        """
         try:
             self.api.shutdown()
         except Exception:
@@ -156,18 +225,49 @@ class Patroni(AbstractPatroniDaemon):
 
 
 def patroni_main(configfile: str) -> None:
-    from multiprocessing import freeze_support
+    """Configure and start ``patroni`` main daemon process.
 
-    freeze_support()
+    :param configfile: path to Patroni configuration file.
+    """
     abstract_main(Patroni, configfile)
 
 
 def process_arguments() -> Namespace:
+    """Process command-line arguments.
+
+    Create a basic command-line parser through :func:`~patroni.daemon.get_base_arg_parser`, extend its capabilities by
+    adding these flags and parse command-line arguments.:
+
+      * ``--validate-config`` -- used to validate the Patroni configuration file
+      * ``--generate-config`` -- used to generate Patroni configuration from a running PostgreSQL instance
+      * ``--generate-sample-config`` -- used to generate a sample Patroni configuration
+
+    .. note::
+        If running with ``--generate-config``, ``--generate-sample-config`` or ``--validate-flag`` will exit
+        after generating or validating configuration.
+
+    :returns: parsed arguments, if not running with ``--validate-config`` flag.
+    """
+    from patroni.config_generator import generate_config
+
     parser = get_base_arg_parser()
-    parser.add_argument('--validate-config', action='store_true', help='Run config validator and exit')
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument('--validate-config', action='store_true', help='Run config validator and exit')
+    group.add_argument('--generate-sample-config', action='store_true',
+                       help='Generate a sample Patroni yaml configuration file')
+    group.add_argument('--generate-config', action='store_true',
+                       help='Generate a Patroni yaml configuration file for a running instance')
+    parser.add_argument('--dsn', help='Optional DSN string of the instance to be used as a source \
+                                    for config generation. Superuser connection is required.')
     args = parser.parse_args()
 
-    if args.validate_config:
+    if args.generate_sample_config:
+        generate_config(args.configfile, True, None)
+        sys.exit(0)
+    elif args.generate_config:
+        generate_config(args.configfile, False, args.dsn)
+        sys.exit(0)
+    elif args.validate_config:
         from patroni.validator import schema
         from patroni.config import Config, ConfigParseError
 
@@ -180,13 +280,66 @@ def process_arguments() -> Namespace:
     return args
 
 
-def main() -> None:
+def check_psycopg() -> None:
-    from patroni import check_psycopg
+    """Ensure at least one among :mod:`psycopg2` or :mod:`psycopg` libraries are available in the environment.
 
-    args = process_arguments()
+    .. note::
+        Patroni chooses :mod:`psycopg2` over :mod:`psycopg`, if possible.
+
+        If nothing meeting the requirements is found, then exit with a fatal message.
+    """
+    min_psycopg2_str = '.'.join(map(str, MIN_PSYCOPG2))
+    min_psycopg3_str = '.'.join(map(str, MIN_PSYCOPG3))
+
+    available_versions: List[str] = []
+
+    # try psycopg2
+    try:
+        from psycopg2 import __version__
+        if parse_version(__version__) >= MIN_PSYCOPG2:
+            return
+        available_versions.append('psycopg2=={0}'.format(__version__.split(' ')[0]))
+    except ImportError:
+        logger.debug('psycopg2 module is not available')
+
+    # try psycopg3
+    try:
+        from psycopg import __version__
+        if parse_version(__version__) >= MIN_PSYCOPG3:
+            return
+        available_versions.append('psycopg=={0}'.format(__version__.split(' ')[0]))
+    except ImportError:
+        logger.debug('psycopg module is not available')
+
+    error = f'FATAL: Patroni requires psycopg2>={min_psycopg2_str}, psycopg2-binary, or psycopg>={min_psycopg3_str}'
+    if available_versions:
+        error += ', but only {0} {1} available'.format(
+            ' and '.join(available_versions),
+            'is' if len(available_versions) == 1 else 'are')
+    sys.exit(error)
+
+
+def main() -> None:
+    """Main entrypoint of :mod:`patroni.__main__`.
+
+    Process command-line arguments, ensure :mod:`psycopg2` (or :mod:`psycopg`) attendee the pre-requisites and start
+    ``patroni`` daemon process.
+
+    .. note::
+        If running through a Docker container, make the main process take care of init process duties and run
+        ``patroni`` daemon as another process. In that case relevant signals received by the main process and forwarded
+        to ``patroni`` daemon process.
+    """
+    from multiprocessing import freeze_support
+
+    # Executables created by PyInstaller are frozen, thus we need to enable frozen support for
+    # :mod:`multiprocessing` to avoid :class:`RuntimeError` exceptions.
+    freeze_support()
 
     check_psycopg()
 
+    args = process_arguments()
+
     if os.getpid() != 1:
         return patroni_main(args.configfile)
 
@@ -196,7 +349,13 @@ def main() -> None:
 
     # Looks like we are in a docker, so we will act like init
     def sigchld_handler(signo: int, stack_frame: Optional[FrameType]) -> None:
+        """Handle ``SIGCHLD`` received by main process from ``patroni`` daemon when the daemon terminates.
+
+        :param signo: signal number.
+        :param stack_frame: current stack frame.
+        """
         try:
+            # log exit code of all children processes, and break loop when there is none left
             while True:
                 ret = os.waitpid(-1, os.WNOHANG)
                 if ret == (0, 0):
@@ -206,7 +365,12 @@ def main() -> None:
         except OSError:
             pass
 
-    def passtochild(signo: int, stack_frame: Optional[FrameType]):
+    def passtochild(signo: int, stack_frame: Optional[FrameType]) -> None:
+        """Forward a signal *signo* from main process to child process.
+
+        :param signo: signal number.
+        :param stack_frame: current stack frame.
+        """
         if pid:
             os.kill(pid, signo)
 

patroni/api.py

@@ -26,7 +26,7 @@ from urllib.parse import urlparse, parse_qs
 
 from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union
 
-from . import psycopg
+from . import global_config, psycopg
 from .__main__ import Patroni
 from .dcs import Cluster
 from .exceptions import PostgresConnectionException, PostgresException
@@ -37,7 +37,7 @@ from .utils import deep_compare, enable_keepalive, parse_bool, patch_config, Ret
 logger = logging.getLogger(__name__)
 
 
-def check_access(func: Callable[['RestApiHandler'], None]) -> Callable[..., None]:
+def check_access(func: Callable[..., None]) -> Callable[..., None]:
     """Check the source ip, authorization header, or client certificates.
 
     .. note::
@@ -49,9 +49,23 @@ def check_access(func: Callable[['RestApiHandler'], None]) -> Callable[..., None
 
     :Example:
 
-        @check_access
+        >>> class FooServer:
-        def do_PUT_foo():
+        ...   def check_access(self, *args, **kwargs):
-            pass
+        ...     print(f'In FooServer: {args[0].__class__.__name__}')
+        ...     return True
+        ...
+
+        >>> class Foo:
+        ...   server = FooServer()
+        ...   @check_access
+        ...   def do_PUT_foo(self):
+        ...      print('In do_PUT_foo')
+
+        >>> f = Foo()
+        >>> f.do_PUT_foo()
+        In FooServer: Foo
+        In do_PUT_foo
+
     """
 
     def wrapper(self: 'RestApiHandler', *args: Any, **kwargs: Any) -> None:
@@ -89,7 +103,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         if TYPE_CHECKING:  # pragma: no cover
             assert isinstance(server, RestApiServer)
         super(RestApiHandler, self).__init__(request, client_address, server)
-        self.server: 'RestApiServer' = server
+        self.server: 'RestApiServer' = server  # pyright: ignore [reportIncompatibleVariableOverride]
         self.__start_time: float = 0.0
         self.path_query: Dict[str, List[str]] = {}
 
@@ -97,6 +111,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         """Write a response that is composed only of the HTTP status.
 
         The response is written with these values separated by space:
+
             * HTTP protocol version;
             * *status_code*;
             * description of *status_code*.
@@ -157,19 +172,21 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Modifies *response* before sending it to the client. Defines the ``patroni`` key, which is a
         dictionary that contains the mandatory keys:
 
-        * ``version``: Patroni version, e.g. ``3.0.2``;
+            * ``version``: Patroni version, e.g. ``3.0.2``;
-        * ``scope``: value of ``scope`` setting from Patroni configuration.
+            * ``scope``: value of ``scope`` setting from Patroni configuration.
 
         May also add the following optional keys, depending on the status of this Patroni/PostgreSQL node:
 
-        * ``tags``: tags that were set through Patroni configuration merged with dynamically applied tags;
+            * ``tags``: tags that were set through Patroni configuration merged with dynamically applied tags;
-        * ``database_system_identifier``: ``Database system identifier`` from ``pg_controldata`` output;
+            * ``database_system_identifier``: ``Database system identifier`` from ``pg_controldata`` output;
-        * ``pending_restart``: ``True`` if PostgreSQL is pending to be restarted;
+            * ``pending_restart``: ``True`` if PostgreSQL is pending to be restarted;
-        * ``scheduled_restart``: a dictionary with a single key ``schedule``, which is the timestamp for the scheduled
+            * ``pending_restart_reason``: dictionary where each key is the parameter that caused "pending restart" flag
-            restart;
+                to be set and the value is a dictionary with the old and the new value.
-        * ``watchdog_failed``: ``True`` if watchdog device is unhealthy;
+            * ``scheduled_restart``: a dictionary with a single key ``schedule``, which is the timestamp for the
-        * ``logger_queue_size``: log queue length if it is longer than expected;
+                scheduled restart;
-        * ``logger_records_lost``: number of log records that have been lost while the log queue was full.
+            * ``watchdog_failed``: ``True`` if watchdog device is unhealthy;
+            * ``logger_queue_size``: log queue length if it is longer than expected;
+            * ``logger_records_lost``: number of log records that have been lost while the log queue was full.
 
         :param status_code: response HTTP status code.
         :param response: represents the status of the PostgreSQL node, and is used as a basis for the HTTP response.
@@ -181,9 +198,14 @@ class RestApiHandler(BaseHTTPRequestHandler):
             response['tags'] = tags
         if patroni.postgresql.sysid:
             response['database_system_identifier'] = patroni.postgresql.sysid
-        if patroni.postgresql.pending_restart:
+        if patroni.postgresql.pending_restart_reason:
             response['pending_restart'] = True
-        response['patroni'] = {'version': patroni.version, 'scope': patroni.postgresql.scope}
+            response['pending_restart_reason'] = dict(patroni.postgresql.pending_restart_reason)
+        response['patroni'] = {
+            'version': patroni.version,
+            'scope': patroni.postgresql.scope,
+            'name': patroni.postgresql.name
+        }
         if patroni.scheduled_restart:
             response['scheduled_restart'] = patroni.scheduled_restart.copy()
             del response['scheduled_restart']['postmaster_start_time']
@@ -204,32 +226,62 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Is used for handling all health-checks requests. E.g. "GET /(primary|replica|sync|async|etc...)".
 
         The (optional) query parameters and the HTTP response status depend on the requested path:
+
             * ``/``, ``primary``, or ``read-write``:
+
                 * HTTP status ``200``: if a primary with the leader lock.
+
             * ``/standby-leader``:
+
                 * HTTP status ``200``: if holds the leader lock in a standby cluster.
+
             * ``/leader``:
+
                 * HTTP status ``200``: if holds the leader lock.
+
             * ``/replica``:
+
                 * Query parameters:
+
                     * ``lag``: only accept replication lag up to ``lag``. Accepts either an :class:`int`, which
                         represents lag in bytes, or a :class:`str` representing lag in human-readable format (e.g.
                         ``10MB``).
                     * Any custom parameter: will attempt to match them against node tags.
+
                 * HTTP status ``200``: if up and running as a standby and without ``noloadbalance`` tag.
+
             * ``/read-only``:
+
                 * HTTP status ``200``: if up and running and without ``noloadbalance`` tag.
+
+            * ``/quorum``:
+
+                * HTTP status ``200``: if up and running as a quorum synchronous standby.
+
+            * ``/read-only-quorum``:
+
+                * HTTP status ``200``: if up and running as a quorum synchronous standby or primary.
+
             * ``/synchronous`` or ``/sync``:
+
                 * HTTP status ``200``: if up and running as a synchronous standby.
+
             * ``/read-only-sync``:
+
                 * HTTP status ``200``: if up and running as a synchronous standby or primary.
+
             * ``/asynchronous``:
+
                 * Query parameters:
+
                     * ``lag``: only accept replication lag up to ``lag``. Accepts either an :class:`int`, which
                         represents lag in bytes, or a :class:`str` representing lag in human-readable format (e.g.
                         ``10MB``).
+
                 * HTTP status ``200``: if up and running as an asynchronous standby.
+
             * ``/health``:
+
                 * HTTP status ``200``: if up and running.
 
         .. note::
@@ -249,7 +301,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         patroni = self.server.patroni
         cluster = patroni.dcs.cluster
-        global_config = patroni.config.get_global_config(cluster)
+        config = global_config.from_cluster(cluster)
 
         leader_optime = cluster and cluster.last_lsn or 0
         replayed_location = response.get('xlog', {}).get('replayed_location', 0)
@@ -267,7 +319,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
             standby_leader_status_code = 200 if response.get('role') == 'standby_leader' else 503
         elif patroni.ha.is_leader():
             leader_status_code = 200
-            if global_config.is_standby_cluster:
+            if config.is_standby_cluster:
                 primary_status_code = replica_status_code = 503
                 standby_leader_status_code = 200 if response.get('role') in ('replica', 'standby_leader') else 503
             else:
@@ -290,16 +342,24 @@ class RestApiHandler(BaseHTTPRequestHandler):
             ignore_tags = True
         elif 'replica' in path:
             status_code = replica_status_code
-        elif 'read-only' in path and 'sync' not in path:
+        elif 'read-only' in path and 'sync' not in path and 'quorum' not in path:
             status_code = 200 if 200 in (primary_status_code, standby_leader_status_code) else replica_status_code
         elif 'health' in path:
             status_code = 200 if response.get('state') == 'running' else 503
         elif cluster:  # dcs is available
+            is_quorum = response.get('quorum_standby')
             is_synchronous = response.get('sync_standby')
             if path in ('/sync', '/synchronous') and is_synchronous:
                 status_code = replica_status_code
-            elif path in ('/async', '/asynchronous') and not is_synchronous:
+            elif path == '/quorum' and is_quorum:
                 status_code = replica_status_code
+            elif path in ('/async', '/asynchronous') and not is_synchronous and not is_quorum:
+                status_code = replica_status_code
+            elif path == '/read-only-quorum':
+                if 200 in (primary_status_code, standby_leader_status_code):
+                    status_code = 200
+                elif is_quorum:
+                    status_code = replica_status_code
             elif path in ('/read-only-sync', '/read-only-synchronous'):
                 if 200 in (primary_status_code, standby_leader_status_code):
                     status_code = 200
@@ -333,16 +393,16 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_OPTIONS(self) -> None:
         """Handle an ``OPTIONS`` request.
 
-        Write a simple HTTP response that represents the current PostgreSQL status. Send only `200 OK` or
+        Write a simple HTTP response that represents the current PostgreSQL status. Send only ``200 OK`` or
-        `503 Service Unavailable` as a response and nothing more, particularly no headers.
+        ``503 Service Unavailable`` as a response and nothing more, particularly no headers.
         """
         self.do_GET(write_status_code_only=True)
 
     def do_HEAD(self) -> None:
         """Handle a ``HEAD`` request.
 
-        Write a simple HTTP response that represents the current PostgreSQL status. Send only `200 OK` or
+        Write a simple HTTP response that represents the current PostgreSQL status. Send only ``200 OK`` or
-        `503 Service Unavailable` as a response and nothing more, particularly no headers.
+        ``503 Service Unavailable`` as a response and nothing more, particularly no headers.
         """
         self.do_GET(write_status_code_only=True)
 
@@ -350,11 +410,17 @@ class RestApiHandler(BaseHTTPRequestHandler):
         """Handle a ``GET`` request to ``/liveness`` path.
 
         Write a simple HTTP response with HTTP status:
+
             * ``200``:
+
                 * If the cluster is in maintenance mode; or
                 * If Patroni heartbeat loop is properly running;
-            * ``503`` if Patroni heartbeat loop last run was more than ``ttl`` setting ago on the primary (or twice the
+
-                value of ``ttl`` on a replica).
+            * ``503``:
+
+                * if Patroni heartbeat loop last run was more than ``ttl`` setting ago on the primary (or twice the
+                    value of ``ttl`` on a replica).
+
         """
         patroni: Patroni = self.server.patroni
         is_primary = patroni.postgresql.role in ('master', 'primary') and patroni.postgresql.is_running()
@@ -371,10 +437,14 @@ class RestApiHandler(BaseHTTPRequestHandler):
         """Handle a ``GET`` request to ``/readiness`` path.
 
         Write a simple HTTP response which HTTP status can be:
+
             * ``200``:
+
                 * If this Patroni node holds the DCS leader lock; or
                 * If this PostgreSQL instance is up and running;
+
             * ``503``: if none of the previous conditions apply.
+
         """
         patroni = self.server.patroni
         if patroni.ha.is_leader():
@@ -397,12 +467,14 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_GET_cluster(self) -> None:
         """Handle a ``GET`` request to ``/cluster`` path.
 
-        Write an HTTP response with JSON content based on the output of :func:`cluster_as_json`, with HTTP status
+        Write an HTTP response with JSON content based on the output of :func:`~patroni.utils.cluster_as_json`, with
-        ``200`` and the JSON representation of the cluster topology.
+        HTTP status ``200`` and the JSON representation of the cluster topology.
         """
-        cluster = self.server.patroni.dcs.get_cluster(True)
+        cluster = self.server.patroni.dcs.get_cluster()
-        global_config = self.server.patroni.config.get_global_config(cluster)
+
-        self._write_json_response(200, cluster_as_json(cluster, global_config))
+        response = cluster_as_json(cluster)
+        response['scope'] = self.server.patroni.postgresql.scope
+        self._write_json_response(200, response)
 
     def do_GET_history(self) -> None:
         """Handle a ``GET`` request to ``/history`` path.
@@ -412,11 +484,13 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         The response contains a :class:`list` of failover/switchover events. Each item is a :class:`list` with the
         following items:
+
             * Timeline when the event occurred (class:`int`);
             * LSN at which the event occurred (class:`int`);
             * The reason for the event (class:`str`);
             * Timestamp when the new timeline was created (class:`str`);
             * Name of the involved Patroni node (class:`str`).
+
         """
         cluster = self.server.patroni.dcs.cluster or self.server.patroni.dcs.get_cluster()
         self._write_json_response(200, cluster.history and cluster.history.lines or [])
@@ -443,26 +517,34 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         The response contains the following items:
 
-        * ``patroni_version``: Patroni version without periods, e.g. ``030002`` for Patroni ``3.0.2``;
+            * ``patroni_version``: Patroni version without periods, e.g. ``030002`` for Patroni ``3.0.2``;
-        * ``patroni_postgres_running``: ``1`` if PostgreSQL is running, else ``0``;
+            * ``patroni_postgres_running``: ``1`` if PostgreSQL is running, else ``0``;
-        * ``patroni_postmaster_start_time``: epoch timestamp since Postmaster was started;
+            * ``patroni_postmaster_start_time``: epoch timestamp since Postmaster was started;
-        * ``patroni_master``: ``1`` if this node holds the leader lock, else ``0``;
+            * ``patroni_master``: ``1`` if this node holds the leader lock, else ``0``;
-        * ``patroni_primary``: same as ``patroni_master``;
+            * ``patroni_primary``: same as ``patroni_master``;
-        * ``patroni_xlog_location``: ``pg_wal_lsn_diff(pg_current_wal_lsn(), '0/0')`` if leader, else ``0``;
+            * ``patroni_xlog_location``: ``pg_wal_lsn_diff(pg_current_wal_flush_lsn(), '0/0')`` if leader, else ``0``;
-        * ``patroni_standby_leader``: ``1`` if standby leader node, else ``0``;
+            * ``patroni_standby_leader``: ``1`` if standby leader node, else ``0``;
-        * ``patroni_replica``: ``1`` if a replica, else ``0``;
+            * ``patroni_replica``: ``1`` if a replica, else ``0``;
-        * ``patroni_sync_standby``: ``1`` if a sync replica, else ``0``;
+            * ``patroni_sync_standby``: ``1`` if a sync replica, else ``0``;
-        * ``patroni_xlog_received_location``: ``pg_wal_lsn_diff(pg_last_wal_receive_lsn(), '0/0')``;
+            * ``patroni_quorum_standby``: ``1`` if a quorum sync replica, else ``0``;
-        * ``patroni_xlog_replayed_location``: ``pg_wal_lsn_diff(pg_last_wal_replay_lsn(), '0/0)``;
+            * ``patroni_xlog_received_location``: ``pg_wal_lsn_diff(pg_last_wal_receive_lsn(), '0/0')``;
-        * ``patroni_xlog_replayed_timestamp``: ``pg_last_xact_replay_timestamp``;
+            * ``patroni_xlog_replayed_location``: ``pg_wal_lsn_diff(pg_last_wal_replay_lsn(), '0/0)``;
-        * ``patroni_xlog_paused``: ``pg_is_wal_replay_paused()``;
+            * ``patroni_xlog_replayed_timestamp``: ``pg_last_xact_replay_timestamp``;
-        * ``patroni_postgres_server_version``: Postgres version without periods, e.g. ``150002`` for Postgres ``15.2``;
+            * ``patroni_xlog_paused``: ``pg_is_wal_replay_paused()``;
-        * ``patroni_cluster_unlocked``: ``1`` if no one holds the leader lock, else ``0``;
+            * ``patroni_postgres_server_version``: Postgres version without periods, e.g. ``150002`` for Postgres
-        * ``patroni_failsafe_mode_is_active``: ``1`` if ``failsafe_mode`` is currently active, else ``0``;
+              ``15.2``;
-        * ``patroni_postgres_timeline``: PostgreSQL timeline based on current WAL file name;
+            * ``patroni_cluster_unlocked``: ``1`` if no one holds the leader lock, else ``0``;
-        * ``patroni_dcs_last_seen``: epoch timestamp when DCS was last contacted successfully;
+            * ``patroni_failsafe_mode_is_active``: ``1`` if ``failsafe_mode`` is currently active, else ``0``;
-        * ``patroni_pending_restart``: ``1`` if this PostgreSQL node is pending a restart, else ``0``;
+            * ``patroni_postgres_timeline``: PostgreSQL timeline based on current WAL file name;
-        * ``patroni_is_paused``: ``1`` if Patroni is in maintenance node, else ``0``.
+            * ``patroni_dcs_last_seen``: epoch timestamp when DCS was last contacted successfully;
+            * ``patroni_pending_restart``: ``1`` if this PostgreSQL node is pending a restart, else ``0``;
+            * ``patroni_is_paused``: ``1`` if Patroni is in maintenance node, else ``0``.
+
+        For PostgreSQL v9.6+ the response will also have the following:
+
+            * ``patroni_postgres_streaming``: 1 if Postgres is streaming from another node, else ``0``;
+            * ``patroni_postgres_in_archive_recovery``: ``1`` if Postgres isn't streaming and
+              there is ``restore_command`` available, else ``0``.
         """
         postgres = self.get_postgresql_status(True)
         patroni = self.server.patroni
@@ -470,113 +552,117 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         metrics: List[str] = []
 
-        scope_label = '{{scope="{0}"}}'.format(patroni.postgresql.scope)
+        labels = f'{{scope="{patroni.postgresql.scope}",name="{patroni.postgresql.name}"}}'
         metrics.append("# HELP patroni_version Patroni semver without periods.")
         metrics.append("# TYPE patroni_version gauge")
         padded_semver = ''.join([x.zfill(2) for x in patroni.version.split('.')])  # 2.0.2 => 020002
-        metrics.append("patroni_version{0} {1}".format(scope_label, padded_semver))
+        metrics.append("patroni_version{0} {1}".format(labels, padded_semver))
 
         metrics.append("# HELP patroni_postgres_running Value is 1 if Postgres is running, 0 otherwise.")
         metrics.append("# TYPE patroni_postgres_running gauge")
-        metrics.append("patroni_postgres_running{0} {1}".format(scope_label, int(postgres['state'] == 'running')))
+        metrics.append("patroni_postgres_running{0} {1}".format(labels, int(postgres['state'] == 'running')))
 
         metrics.append("# HELP patroni_postmaster_start_time Epoch seconds since Postgres started.")
         metrics.append("# TYPE patroni_postmaster_start_time gauge")
         postmaster_start_time = postgres.get('postmaster_start_time')
         postmaster_start_time = (postmaster_start_time - epoch).total_seconds() if postmaster_start_time else 0
-        metrics.append("patroni_postmaster_start_time{0} {1}".format(scope_label, postmaster_start_time))
+        metrics.append("patroni_postmaster_start_time{0} {1}".format(labels, postmaster_start_time))
 
         metrics.append("# HELP patroni_master Value is 1 if this node is the leader, 0 otherwise.")
         metrics.append("# TYPE patroni_master gauge")
-        metrics.append("patroni_master{0} {1}".format(scope_label, int(postgres['role'] in ('master', 'primary'))))
+        metrics.append("patroni_master{0} {1}".format(labels, int(postgres['role'] in ('master', 'primary'))))
 
         metrics.append("# HELP patroni_primary Value is 1 if this node is the leader, 0 otherwise.")
         metrics.append("# TYPE patroni_primary gauge")
-        metrics.append("patroni_primary{0} {1}".format(scope_label, int(postgres['role'] in ('master', 'primary'))))
+        metrics.append("patroni_primary{0} {1}".format(labels, int(postgres['role'] in ('master', 'primary'))))
 
         metrics.append("# HELP patroni_xlog_location Current location of the Postgres"
                        " transaction log, 0 if this node is not the leader.")
         metrics.append("# TYPE patroni_xlog_location counter")
-        metrics.append("patroni_xlog_location{0} {1}".format(scope_label, postgres.get('xlog', {}).get('location', 0)))
+        metrics.append("patroni_xlog_location{0} {1}".format(labels, postgres.get('xlog', {}).get('location', 0)))
 
         metrics.append("# HELP patroni_standby_leader Value is 1 if this node is the standby_leader, 0 otherwise.")
         metrics.append("# TYPE patroni_standby_leader gauge")
-        metrics.append("patroni_standby_leader{0} {1}".format(scope_label, int(postgres['role'] == 'standby_leader')))
+        metrics.append("patroni_standby_leader{0} {1}".format(labels, int(postgres['role'] == 'standby_leader')))
 
         metrics.append("# HELP patroni_replica Value is 1 if this node is a replica, 0 otherwise.")
         metrics.append("# TYPE patroni_replica gauge")
-        metrics.append("patroni_replica{0} {1}".format(scope_label, int(postgres['role'] == 'replica')))
+        metrics.append("patroni_replica{0} {1}".format(labels, int(postgres['role'] == 'replica')))
 
-        metrics.append("# HELP patroni_sync_standby Value is 1 if this node is a sync standby replica, 0 otherwise.")
+        metrics.append("# HELP patroni_sync_standby Value is 1 if this node is a sync standby, 0 otherwise.")
         metrics.append("# TYPE patroni_sync_standby gauge")
-        metrics.append("patroni_sync_standby{0} {1}".format(scope_label, int(postgres.get('sync_standby', False))))
+        metrics.append("patroni_sync_standby{0} {1}".format(labels, int(postgres.get('sync_standby', False))))
+
+        metrics.append("# HELP patroni_quorum_standby Value is 1 if this node is a quorum standby, 0 otherwise.")
+        metrics.append("# TYPE patroni_quorum_standby gauge")
+        metrics.append("patroni_quorum_standby{0} {1}".format(labels, int(postgres.get('quorum_standby', False))))
 
         metrics.append("# HELP patroni_xlog_received_location Current location of the received"
                        " Postgres transaction log, 0 if this node is not a replica.")
         metrics.append("# TYPE patroni_xlog_received_location counter")
         metrics.append("patroni_xlog_received_location{0} {1}"
-                       .format(scope_label, postgres.get('xlog', {}).get('received_location', 0)))
+                       .format(labels, postgres.get('xlog', {}).get('received_location', 0)))
 
         metrics.append("# HELP patroni_xlog_replayed_location Current location of the replayed"
                        " Postgres transaction log, 0 if this node is not a replica.")
         metrics.append("# TYPE patroni_xlog_replayed_location counter")
         metrics.append("patroni_xlog_replayed_location{0} {1}"
-                       .format(scope_label, postgres.get('xlog', {}).get('replayed_location', 0)))
+                       .format(labels, postgres.get('xlog', {}).get('replayed_location', 0)))
 
         metrics.append("# HELP patroni_xlog_replayed_timestamp Current timestamp of the replayed"
                        " Postgres transaction log, 0 if null.")
         metrics.append("# TYPE patroni_xlog_replayed_timestamp gauge")
         replayed_timestamp = postgres.get('xlog', {}).get('replayed_timestamp')
         replayed_timestamp = (replayed_timestamp - epoch).total_seconds() if replayed_timestamp else 0
-        metrics.append("patroni_xlog_replayed_timestamp{0} {1}".format(scope_label, replayed_timestamp))
+        metrics.append("patroni_xlog_replayed_timestamp{0} {1}".format(labels, replayed_timestamp))
 
         metrics.append("# HELP patroni_xlog_paused Value is 1 if the Postgres xlog is paused, 0 otherwise.")
         metrics.append("# TYPE patroni_xlog_paused gauge")
         metrics.append("patroni_xlog_paused{0} {1}"
-                       .format(scope_label, int(postgres.get('xlog', {}).get('paused', False) is True)))
+                       .format(labels, int(postgres.get('xlog', {}).get('paused', False) is True)))
 
         if postgres.get('server_version', 0) >= 90600:
             metrics.append("# HELP patroni_postgres_streaming Value is 1 if Postgres is streaming, 0 otherwise.")
             metrics.append("# TYPE patroni_postgres_streaming gauge")
             metrics.append("patroni_postgres_streaming{0} {1}"
-                           .format(scope_label, int(postgres.get('replication_state') == 'streaming')))
+                           .format(labels, int(postgres.get('replication_state') == 'streaming')))
 
             metrics.append("# HELP patroni_postgres_in_archive_recovery Value is 1"
                            " if Postgres is replicating from archive, 0 otherwise.")
             metrics.append("# TYPE patroni_postgres_in_archive_recovery gauge")
             metrics.append("patroni_postgres_in_archive_recovery{0} {1}"
-                           .format(scope_label, int(postgres.get('replication_state') == 'in archive recovery')))
+                           .format(labels, int(postgres.get('replication_state') == 'in archive recovery')))
 
         metrics.append("# HELP patroni_postgres_server_version Version of Postgres (if running), 0 otherwise.")
         metrics.append("# TYPE patroni_postgres_server_version gauge")
-        metrics.append("patroni_postgres_server_version {0} {1}".format(scope_label, postgres.get('server_version', 0)))
+        metrics.append("patroni_postgres_server_version {0} {1}".format(labels, postgres.get('server_version', 0)))
 
         metrics.append("# HELP patroni_cluster_unlocked Value is 1 if the cluster is unlocked, 0 if locked.")
         metrics.append("# TYPE patroni_cluster_unlocked gauge")
-        metrics.append("patroni_cluster_unlocked{0} {1}".format(scope_label, int(postgres.get('cluster_unlocked', 0))))
+        metrics.append("patroni_cluster_unlocked{0} {1}".format(labels, int(postgres.get('cluster_unlocked', 0))))
 
         metrics.append("# HELP patroni_failsafe_mode_is_active Value is 1 if failsafe mode is active, 0 if inactive.")
         metrics.append("# TYPE patroni_failsafe_mode_is_active gauge")
         metrics.append("patroni_failsafe_mode_is_active{0} {1}"
-                       .format(scope_label, int(postgres.get('failsafe_mode_is_active', 0))))
+                       .format(labels, int(postgres.get('failsafe_mode_is_active', 0))))
 
         metrics.append("# HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise.")
         metrics.append("# TYPE patroni_postgres_timeline counter")
-        metrics.append("patroni_postgres_timeline{0} {1}".format(scope_label, postgres.get('timeline', 0)))
+        metrics.append("patroni_postgres_timeline{0} {1}".format(labels, postgres.get('timeline', 0)))
 
         metrics.append("# HELP patroni_dcs_last_seen Epoch timestamp when DCS was last contacted successfully"
                        " by Patroni.")
         metrics.append("# TYPE patroni_dcs_last_seen gauge")
-        metrics.append("patroni_dcs_last_seen{0} {1}".format(scope_label, postgres.get('dcs_last_seen', 0)))
+        metrics.append("patroni_dcs_last_seen{0} {1}".format(labels, postgres.get('dcs_last_seen', 0)))
 
         metrics.append("# HELP patroni_pending_restart Value is 1 if the node needs a restart, 0 otherwise.")
         metrics.append("# TYPE patroni_pending_restart gauge")
         metrics.append("patroni_pending_restart{0} {1}"
-                       .format(scope_label, int(patroni.postgresql.pending_restart)))
+                       .format(labels, int(bool(patroni.postgresql.pending_restart_reason))))
 
         metrics.append("# HELP patroni_is_paused Value is 1 if auto failover is disabled, 0 otherwise.")
         metrics.append("# TYPE patroni_is_paused gauge")
-        metrics.append("patroni_is_paused{0} {1}".format(scope_label, int(postgres.get('pause', 0))))
+        metrics.append("patroni_is_paused{0} {1}".format(labels, int(postgres.get('pause', 0))))
 
         self.write_response(200, '\n'.join(metrics) + '\n', content_type='text/plain')
 
@@ -627,7 +713,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         """
         request = self._read_json_content()
         if request:
-            cluster = self.server.patroni.dcs.get_cluster(True)
+            cluster = self.server.patroni.dcs.get_cluster()
             if not (cluster.config and cluster.config.modify_version):
                 return self.send_error(503)
             data = cluster.config.data.copy()
@@ -661,7 +747,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_POST_reload(self) -> None:
         """Handle a ``POST`` request to ``/reload`` path.
 
-        Schedules a reload to Patroni and writes a response with HTTP status `202`.
+        Schedules a reload to Patroni and writes a response with HTTP status ``202``.
         """
         self.server.patroni.sighup_handler()
         self.write_response(202, 'reload scheduled')
@@ -722,13 +808,17 @@ class RestApiHandler(BaseHTTPRequestHandler):
         :param schedule: a string representing a timestamp, e.g. ``2023-04-14T20:27:00+00:00``.
         :param action: the action to be scheduled (``restart``, ``switchover``, or ``failover``).
 
-        :returns: a tuple composed of 3 items
+        :returns: a tuple composed of 3 items:
+
             * Suggested HTTP status code for a response:
+
                 * ``None``: if no issue was faced while parsing, leaving it up to the caller to decide the status; or
                 * ``400``: if no timezone information could be found in *schedule*; or
                 * ``422``: if *schedule* is invalid -- in the past or not parsable.
+
             * An error message, if any error is faced, otherwise ``None``;
             * Parsed *schedule*, if able to parse, otherwise ``None``.
+
         """
         error = None
         scheduled_at = None
@@ -755,25 +845,31 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Used to restart postgres (or schedule a restart), mainly by ``patronictl restart``.
 
         The request body should be a JSON dictionary, and it can contain the following keys:
+
             * ``schedule``: timestamp at which the restart should occur;
             * ``role``: restart only nodes which role is ``role``. Can be either:
+
                 * ``primary`` (or ``master``); or
                 * ``replica``.
+
             * ``postgres_version``: restart only nodes which PostgreSQL version is less than ``postgres_version``, e.g.
                 ``15.2``;
             * ``timeout``: if restart takes longer than ``timeout`` return an error and fail over to a replica;
             * ``restart_pending``: if we should restart only when have ``pending restart`` flag;
 
         Response HTTP status codes:
+
             * ``200``: if successfully performed an immediate restart; or
             * ``202``: if successfully scheduled a restart for later; or
             * ``500``: if the cluster is in maintenance mode; or
             * ``400``: if
+
                 * ``role`` value is invalid; or
                 * ``postgres_version`` value is invalid; or
                 * ``timeout`` is not a number, or lesser than ``0``; or
                 * request contains an unknown key; or
                 * exception is faced while performing an immediate restart.
+
             * ``409``: if another restart was already previously scheduled; or
             * ``503``: if any issue was found while performing an immediate restart; or
             * HTTP status returned by :func:`parse_schedule`, if any error was observed while parsing the schedule.
@@ -791,7 +887,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         if request:
             logger.debug("received restart request: {0}".format(request))
 
-        if self.server.patroni.config.get_global_config(cluster).is_paused and 'schedule' in request:
+        if global_config.from_cluster(cluster).is_paused and 'schedule' in request:
             self.write_response(status_code, "Can't schedule restart in the paused state")
             return
 
@@ -851,6 +947,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Used to remove a scheduled restart of PostgreSQL.
 
         Response HTTP status codes:
+
             * ``200``: if a scheduled restart was removed; or
             * ``404``: if no scheduled restart could be found.
         """
@@ -869,6 +966,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Used to remove a scheduled switchover in the cluster.
 
         It writes a response, and the HTTP status code can be:
+
             * ``200``: if a scheduled switchover was removed; or
             * ``404``: if no scheduled switchover could be found; or
             * ``409``: if not able to update the switchover info in the DCS.
@@ -890,11 +988,13 @@ class RestApiHandler(BaseHTTPRequestHandler):
         """Handle a ``POST`` request to ``/reinitialize`` path.
 
         The request body may contain a JSON dictionary with the following key:
+
             * ``force``: ``True`` if we want to cancel an already running task in order to reinit a replica.
 
         Response HTTP status codes:
+
             * ``200``: if the reinit operation has started; or
-            * ``503``: if any error is returned by :func:`Ha.reinitialize`.
+            * ``503``: if any error is returned by :func:`~patroni.ha.Ha.reinitialize`.
         """
         request = self._read_json_content(body_is_optional=True)
 
@@ -918,11 +1018,15 @@ class RestApiHandler(BaseHTTPRequestHandler):
         :param candidate: name of the Patroni node to be promoted.
         :param action: the action that is ongoing (``switchover`` or ``failover``).
 
-        :returns: a tuple composed of 2 items
+        :returns: a tuple composed of 2 items:
+
             * Response HTTP status codes:
+
                 * ``200``: if the operation succeeded; or
                 * ``503``: if the operation failed or timed out.
+
             * A status message about the operation.
+
         """
         timeout = max(10, self.server.patroni.dcs.loop_wait)
         for _ in range(0, timeout * 2):
@@ -952,16 +1056,17 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         :returns: a string with the error message or ``None`` if good nodes are found.
         """
-        is_synchronous_mode = self.server.patroni.config.get_global_config(cluster).is_synchronous_mode
+        config = global_config.from_cluster(cluster)
         if leader and (not cluster.leader or cluster.leader.name != leader):
             return 'leader name does not match'
         if candidate:
-            if action == 'switchover' and is_synchronous_mode and not cluster.sync.matches(candidate):
+            if action == 'switchover' and config.is_synchronous_mode\
+                    and not config.is_quorum_commit_mode and not cluster.sync.matches(candidate):
                 return 'candidate name does not match with sync_standby'
             members = [m for m in cluster.members if m.name == candidate]
             if not members:
                 return 'candidate does not exists'
-        elif is_synchronous_mode:
+        elif config.is_synchronous_mode and not config.is_quorum_commit_mode:
             members = [m for m in cluster.members if cluster.sync.matches(m.name)]
             if not members:
                 return action + ' is not possible: can not find sync_standby'
@@ -981,17 +1086,20 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Handles manual failovers/switchovers, mainly from ``patronictl``.
 
         The request body should be a JSON dictionary, and it can contain the following keys:
+
             * ``leader``: name of the current leader in the cluster;
             * ``candidate``: name of the Patroni node to be promoted;
             * ``scheduled_at``: a string representing the timestamp when to execute the switchover/failover, e.g.
                 ``2023-04-14T20:27:00+00:00``.
 
         Response HTTP status codes:
+
             * ``202``: if operation has been scheduled;
             * ``412``: if operation is not possible;
             * ``503``: if unable to register the operation to the DCS;
             * HTTP status returned by :func:`parse_schedule`, if any error was observed while parsing the schedule;
-            * HTTP status returned by :func:`poll_failover_result` if the operation has been processed immediately.
+            * HTTP status returned by :func:`poll_failover_result` if the operation has been processed immediately;
+            * ``400``: if none of the above applies.
 
         .. note::
             If unable to parse the request body, then the request is silently discarded.
@@ -1007,7 +1115,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         candidate = request.get('candidate') or request.get('member')
         scheduled_at = request.get('scheduled_at')
         cluster = self.server.patroni.dcs.get_cluster()
-        global_config = self.server.patroni.config.get_global_config(cluster)
+        config = global_config.from_cluster(cluster)
 
         logger.info("received %s request with leader=%s candidate=%s scheduled_at=%s",
                     action, leader, candidate, scheduled_at)
@@ -1018,15 +1126,22 @@ class RestApiHandler(BaseHTTPRequestHandler):
             data = 'Switchover could be performed only from a specific leader'
 
         if not data and scheduled_at:
-            if not leader:
+            if action == 'failover':
-                data = 'Scheduled {0} is possible only from a specific leader'.format(action)
+                data = "Failover can't be scheduled"
-            if not data and global_config.is_paused:
+            elif config.is_paused:
-                data = "Can't schedule {0} in the paused state".format(action)
+                data = "Can't schedule switchover in the paused state"
-            if not data:
+            else:
                 (status_code, data, scheduled_at) = self.parse_schedule(scheduled_at, action)
 
-        if not data and global_config.is_paused and not candidate:
+        if not data and config.is_paused and not candidate:
-            data = action.title() + ' is possible only to a specific candidate in a paused state'
+            data = 'Switchover is possible only to a specific candidate in a paused state'
+
+        if action == 'failover' and leader:
+            logger.warning('received failover request with leader specifed - performing switchover instead')
+            action = 'switchover'
+
+        if not data and leader == candidate:
+            data = 'Switchover target and source are the same'
 
         if not data and not scheduled_at:
             data = self.is_failover_possible(cluster, leader, candidate, action)
@@ -1043,7 +1158,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name,
                                                                   candidate, action)
             else:
-                data = 'failed to write {0} key into DCS'.format(action)
+                data = 'failed to write failover key into DCS'
                 status_code = 503
         # pyright thinks ``status_code`` can be ``None`` because ``parse_schedule`` call may return ``None``. However,
         # if that's the case, ``status_code`` will be overwritten somewhere between ``parse_schedule`` and
@@ -1063,8 +1178,16 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_POST_citus(self) -> None:
         """Handle a ``POST`` request to ``/citus`` path.
 
-        Call :func:`CitusHandler.handle_event` to handle the request, then write a response with HTTP status code
+        .. note::
-        ``200``.
+            We keep this entrypoint for backward compatibility and simply dispatch the request to :meth:`do_POST_mpp`.
+        """
+        self.do_POST_mpp()
+
+    def do_POST_mpp(self) -> None:
+        """Handle a ``POST`` request to ``/mpp`` path.
+
+        Call :func:`~patroni.postgresql.mpp.AbstractMPPHandler.handle_event` to handle the request,
+        then write a response with HTTP status code ``200``.
 
         .. note::
             If unable to parse the request body, then the request is silently discarded.
@@ -1074,24 +1197,27 @@ class RestApiHandler(BaseHTTPRequestHandler):
             return
 
         patroni = self.server.patroni
-        if patroni.postgresql.citus_handler.is_coordinator() and patroni.ha.is_leader():
+        if patroni.postgresql.mpp_handler.is_coordinator() and patroni.ha.is_leader():
-            cluster = patroni.dcs.get_cluster(True)
+            cluster = patroni.dcs.get_cluster()
-            patroni.postgresql.citus_handler.handle_event(cluster, request)
+            patroni.postgresql.mpp_handler.handle_event(cluster, request)
         self.write_response(200, 'OK')
 
     def parse_request(self) -> bool:
-        """Override :func:`parse_request` method to enrich basic functionality of :class:`BaseHTTPRequestHandler`.
+        """Override :func:`parse_request` to enrich basic functionality of :class:`~http.server.BaseHTTPRequestHandler`.
 
         Original class can only invoke :func:`do_GET`, :func:`do_POST`, :func:`do_PUT`, etc method implementations if
         they are defined.
+
         But we would like to have at least some simple routing mechanism, i.e.:
+
             * ``GET /uri1/part2`` request should invoke :func:`do_GET_uri1()`
             * ``POST /other`` should invoke :func:`do_POST_other()`
 
         If the :func:`do_<REQUEST_METHOD>_<first_part_url>` method does not exist we'll fall back to original behavior.
 
         :returns: ``True`` for success, ``False`` for failure; on failure, any relevant error response has already been
-        sent back.
+                  sent back.
+
         """
         ret = BaseHTTPRequestHandler.parse_request(self)
         if ret:
@@ -1104,20 +1230,18 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 self.command = mname
         return ret
 
-    def query(self, sql: str, *params: Any, **kwargs: Any) -> List[Tuple[Any, ...]]:
+    def query(self, sql: str, *params: Any, retry: bool = False) -> List[Tuple[Any, ...]]:
-        """Execute *sql* query with *params*.
+        """Execute *sql* query with *params* and optionally return results.
 
         :param sql: the SQL statement to be run.
         :param params: positional arguments to call :func:`RestApiServer.query` with.
-        :param kwargs: can contain the key ``retry``. If the key is present its value should be a :class:`bool` which
+        :param retry: whether the query should be retried upon failure or given up immediately.
-            indicates whether the query should be retried upon failure or given up immediately.
 
         :returns: a list of rows that were fetched from the database.
         """
-        if not kwargs.get('retry', False):
+        if not retry:
             return self.server.query(sql, *params)
-        retry = Retry(delay=1, retry_exceptions=PostgresConnectionException)
+        return Retry(delay=1, retry_exceptions=PostgresConnectionException)(self.server.query, sql, *params)
-        return retry(self.server.query, sql, *params)
 
     def get_postgresql_status(self, retry: bool = False) -> Dict[str, Any]:
         """Builds an object representing a status of "postgres".
@@ -1125,40 +1249,51 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Some of the values are collected by executing a query and other are taken from the state stored in memory.
 
         :param retry: whether the query should be retried if failed or give up immediately
+
         :returns: a dict with the status of Postgres/Patroni. The keys are:
+
             * ``state``: Postgres state among ``stopping``, ``stopped``, ``stop failed``, ``crashed``, ``running``,
-                ``starting``, ``start failed``, ``restarting``, ``restart failed``, ``initializing new cluster``,
+              ``starting``, ``start failed``, ``restarting``, ``restart failed``, ``initializing new cluster``,
-                ``initdb failed``, ``running custom bootstrap script``, ``custom bootstrap failed``,
+              ``initdb failed``, ``running custom bootstrap script``, ``custom bootstrap failed``,
-                ``creating replica``, or ``unknown``;
+              ``creating replica``, or ``unknown``;
             * ``postmaster_start_time``: ``pg_postmaster_start_time()``;
             * ``role``: ``replica`` or ``master`` based on ``pg_is_in_recovery()`` output;
             * ``server_version``: Postgres version without periods, e.g. ``150002`` for Postgres ``15.2``;
             * ``xlog``: dictionary. Its structure depends on ``role``:
+
                 * If ``master``:
-                    * ``location``: ``pg_current_wal_lsn()``
+
+                    * ``location``: ``pg_current_wal_flush_lsn()``
+
                 * If ``replica``:
+
                     * ``received_location``: ``pg_wal_lsn_diff(pg_last_wal_receive_lsn(), '0/0')``;
                     * ``replayed_location``: ``pg_wal_lsn_diff(pg_last_wal_replay_lsn(), '0/0)``;
                     * ``replayed_timestamp``: ``pg_last_xact_replay_timestamp``;
                     * ``paused``: ``pg_is_wal_replay_paused()``;
+
             * ``sync_standby``: ``True`` if replication mode is synchronous and this is a sync standby;
+            * ``quorum_standby``: ``True`` if replication mode is quorum and this is a quorum standby;
             * ``timeline``: PostgreSQL primary node timeline;
             * ``replication``: :class:`list` of :class:`dict` entries, one for each replication connection. Each entry
                 contains the following keys:
+
                 * ``application_name``: ``pg_stat_activity.application_name``;
                 * ``client_addr``: ``pg_stat_activity.client_addr``;
                 * ``state``: ``pg_stat_replication.state``;
                 * ``sync_priority``: ``pg_stat_replication.sync_priority``;
                 * ``sync_state``: ``pg_stat_replication.sync_state``;
                 * ``usename``: ``pg_stat_activity.usename``.
+
             * ``pause``: ``True`` if cluster is in maintenance mode;
             * ``cluster_unlocked``: ``True`` if cluster has no node holding the leader lock;
             * ``failsafe_mode_is_active``: ``True`` if DCS failsafe mode is currently active;
             * ``dcs_last_seen``: epoch timestamp DCS was last reached by Patroni.
+
         """
         postgresql = self.server.patroni.postgresql
         cluster = self.server.patroni.dcs.cluster
-        global_config = self.server.patroni.config.get_global_config(cluster)
+        config = global_config.from_cluster(cluster)
         try:
 
             if postgresql.state not in ('running', 'restarting', 'starting'):
@@ -1173,8 +1308,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     " application_name, client_addr, w.state, sync_state, sync_priority"
                     " FROM pg_catalog.pg_stat_get_wal_senders() w, pg_catalog.pg_stat_get_activity(pid)) AS ri")
 
-            row = self.query(stmt.format(postgresql.wal_name, postgresql.lsn_name), retry=retry)[0]
+            row = self.query(stmt.format(postgresql.wal_name, postgresql.lsn_name,
-
+                                         postgresql.wal_flush), retry=retry)[0]
             result = {
                 'state': postgresql.state,
                 'postmaster_start_time': row[0],
@@ -1189,12 +1324,12 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 })
             }
 
-            if result['role'] == 'replica' and global_config.is_standby_cluster:
+            if result['role'] == 'replica' and config.is_standby_cluster:
                 result['role'] = postgresql.role
 
-            if result['role'] == 'replica' and global_config.is_synchronous_mode\
+            if result['role'] == 'replica' and config.is_synchronous_mode\
                     and cluster and cluster.sync.matches(postgresql.name):
-                result['sync_standby'] = True
+                result['quorum_standby' if global_config.is_quorum_commit_mode else 'sync_standby'] = True
 
             if row[1] > 0:
                 result['timeline'] = row[1]
@@ -1217,7 +1352,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 state = 'unknown'
             result: Dict[str, Any] = {'state': state, 'role': postgresql.role}
 
-        if global_config.is_paused:
+        if config.is_paused:
             result['pause'] = True
         if not cluster or cluster.is_unlocked():
             result['cluster_unlocked'] = True
@@ -1279,24 +1414,35 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         self.daemon = True
 
     def query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]:
-        """Execute *sql* query with *params*.
+        """Execute *sql* query with *params* and optionally return results.
+
+        .. note::
+            Prefer to use own connection to postgres and fallback to ``heartbeat`` when own isn't available.
 
         :param sql: the SQL statement to be run.
         :param params: positional arguments to be used as parameters for *sql*.
 
         :returns: a list of rows that were fetched from the database.
-        :raises psycopg.Error: if had issues while executing *sql*.
+
-        :raises PostgresConnectionException: if had issues while connecting to the database.
+        :raises:
+            :class:`psycopg.Error`: if had issues while executing *sql*.
+            :class:`~patroni.exceptions.PostgresConnectionException`: if had issues while connecting to the database.
         """
-        cursor = None
+        # We first try to get a heartbeat connection because it is always required for the main thread.
         try:
-            with self.patroni.postgresql.connection().cursor() as cursor:
+            heartbeat_connection = self.patroni.postgresql.connection_pool.get('heartbeat')
-                cursor.execute(sql.encode('utf-8'), params)
+            heartbeat_connection.get()  # try to open psycopg connection to postgres
-                return [r for r in cursor]
+        except psycopg.Error as exc:
-        except psycopg.Error as e:
+            raise PostgresConnectionException('connection problems') from exc
-            if cursor and cursor.connection.closed == 0:
+
-                raise e
+        try:
-            raise PostgresConnectionException('connection problems')
+            connection = self.patroni.postgresql.connection_pool.get('restapi')
+            connection.get()  # try to open psycopg connection to postgres
+        except psycopg.Error:
+            logger.debug('restapi connection to postgres is not available')
+            connection = heartbeat_connection
+
+        return connection.query(sql, *params)
 
     @staticmethod
     def _set_fd_cloexec(fd: socket.socket) -> None:
@@ -1346,7 +1492,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         :param host: hostname to be checked.
         :param port: port to be checked.
 
-        :rtype: Iterator[Union[IPv4Network, IPv6Network]] of *host* + *port* resolved to IP networks.
+        :yields: *host* + *port* resolved to IP networks.
         """
         try:
             for _, _, _, _, sa in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP):
@@ -1360,8 +1506,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         .. note::
             Only yields object if ``restapi.allowlist_include_members`` setting is enabled.
 
-        :rtype: Iterator[Union[IPv4Network, IPv6Network]] of each node ``restapi.connect_address`` resolved to an IP
+        :yields: each node ``restapi.connect_address`` resolved to an IP network.
-            network.
         """
         cluster = self.patroni.dcs.cluster
         if self.__allowlist_include_members and cluster:
@@ -1381,8 +1526,10 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         """Ensure client has enough privileges to perform a given request.
 
         Write a response back to the client if any issue is observed, and the HTTP status may be:
+
             * ``401``: if ``Authorization`` header is missing or contain an invalid password;
             * ``403``: if:
+
                 * ``restapi.allowlist`` was configured, but client IP is not in the allowed list; or
                 * ``restapi.allowlist_include_members`` is enabled, but client IP is not in the members list; or
                 * a client certificate is expected by the server, but is missing in the request.
@@ -1461,18 +1608,21 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         :param listen: IP and port to bind REST API to. It should be a string in the format ``host:port``, where
             ``host`` can be a hostname or IP address. It is the value of ``restapi.listen`` setting.
         :param ssl_options: dictionary that may contain the following keys, depending on what has been configured in
-            ``restapi` section:
+            ``restapi`` section:
+
             * ``certfile``: path to PEM certificate. If given, will start in HTTPS mode;
             * ``keyfile``: path to key of ``certfile``;
             * ``keyfile_password``: password for decrypting ``keyfile``;
             * ``cafile``: path to CA file to validate client certificates;
             * ``ciphers``: permitted cipher suites;
             * ``verify_client``: value can be one among:
+
                 * ``none``: do not check client certificates;
                 * ``optional``: check client certificate only for unsafe REST API endpoints;
                 * ``required``: check client certificate for all REST API endpoints.
 
-        :raises ValueError: if any issue is faced while parsing *listen*.
+        :raises:
+            :class:`ValueError`: if any issue is faced while parsing *listen*.
         """
         try:
             host, port = split_host_port(listen, None)
@@ -1520,7 +1670,8 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
                                client_address: Tuple[str, int]) -> None:
         """Process a request to the REST API.
 
-        Wrapper for :func:`ThreadingMixIn.process_request_thread` that additionally:
+        Wrapper for :func:`~socketserver.ThreadingMixIn.process_request_thread` that additionally:
+
             * Enable TCP keepalive
             * Perform SSL handshake (if an SSL socket).
 
@@ -1538,7 +1689,8 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
     def shutdown_request(self, request: Union[socket.socket, Tuple[bytes, socket.socket]]) -> None:
         """Shut down a request to the REST API.
 
-        Wrapper for :func:`HTTPServer.shutdown_request` that additionally:
+        Wrapper for :func:`http.server.HTTPServer.shutdown_request` that additionally:
+
             * Perform SSL shutdown handshake (if a SSL socket).
 
         :param request: socket to handle the client request.
@@ -1586,7 +1738,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         :param value: list of IPs and/or networks contained in ``restapi.allowlist`` setting. Each item can be a host,
             an IP, or a network in CIDR format.
 
-        :rtype: Iterator[Union[IPv4Network, IPv6Network]] of *host* + *port* resolved to IP networks.
+        :yields: *host* + *port* resolved to IP networks.
         """
         if isinstance(value, list):
             for v in value:
@@ -1603,7 +1755,9 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         """Reload REST API configuration.
 
         :param config: dictionary representing values under the ``restapi`` configuration section.
-        :raises ValueError: if ``listen`` key is not present in *config*.
+
+        :raises:
+            :class:`ValueError`: if ``listen`` key is not present in *config*.
         """
         if 'listen' not in config:  # changing config in runtime
             raise ValueError('Can not find "restapi.listen" config')

patroni/collections.py

@@ -3,7 +3,7 @@
 Provides a case insensitive :class:`dict` and :class:`set` object types.
 """
 from collections import OrderedDict
-from typing import Any, Collection, Dict, Iterator, MutableMapping, MutableSet, Optional
+from typing import Any, Collection, Dict, Iterator, KeysView, MutableMapping, MutableSet, Optional
 
 
 class CaseInsensitiveSet(MutableSet[str]):
@@ -187,6 +187,13 @@ class CaseInsensitiveDict(MutableMapping[str, Any]):
         """
         return CaseInsensitiveDict({v[0]: v[1] for v in self._values.values()})
 
+    def keys(self) -> KeysView[str]:
+        """Return a new view of the dict's keys.
+
+        :returns: a set-like object providing a view on the dict's keys
+        """
+        return self._values.keys()
+
     def __repr__(self) -> str:
         """Get a string representation of the dict.
 

patroni/config.py

@@ -1,3 +1,5 @@
+"""Facilities related to Patroni configuration."""
+import re
 import json
 import logging
 import os
@@ -11,9 +13,11 @@ from typing import Any, Callable, Collection, Dict, List, Optional, Union, TYPE_
 
 from . import PATRONI_ENV_PREFIX
 from .collections import CaseInsensitiveDict
-from .dcs import ClusterConfig, Cluster
+from .dcs import ClusterConfig
 from .exceptions import ConfigParseError
+from .file_perm import pg_perm
 from .postgresql.config import ConfigHandler
+from .validator import IntValidator
 from .utils import deep_compare, parse_bool, parse_int, patch_config
 
 logger = logging.getLogger(__name__)
@@ -34,148 +38,47 @@ _AUTH_ALLOWED_PARAMETERS = (
 
 
 def default_validator(conf: Dict[str, Any]) -> List[str]:
+    """Ensure *conf* is not empty.
+
+    Designed to be used as default validator for :class:`Config` objects, if no specific validator is provided.
+
+    :param conf: configuration to be validated.
+
+    :returns: an empty list -- :class:`Config` expects the validator to return a list of 0 or more issues found while
+        validating the configuration.
+
+    :raises:
+        :class:`ConfigParseError`: if *conf* is empty.
+    """
     if not conf:
         raise ConfigParseError("Config is empty.")
     return []
 
 
-class GlobalConfig(object):
-
-    """A class that wrapps global configuration and provides convinient methods to access/check values.
-
-    It is instantiated by calling :func:`Config.global_config` method which picks either a
-    configuration from provided :class:`Cluster` object (the most up-to-date) or from the
-    local cache if :class::`ClusterConfig` is not initialized or doesn't have a valid config.
-    """
-
-    def __init__(self, config: Dict[str, Any]) -> None:
-        """Initialize :class:`GlobalConfig` object.
-
-        :param config: current configuration either from
-                       :class:`ClusterConfig` or from :class:`Config.dynamic_configuration`
-        """
-        self.__config = config
-
-    def get(self, name: str) -> Any:
-        """Gets global configuration value by name.
-
-        :param name: parameter name
-        :returns: configuration value or `None` if it is missing
-        """
-        return self.__config.get(name)
-
-    def check_mode(self, mode: str) -> bool:
-        """Checks whether the certain parameter is enabled.
-
-        :param mode: parameter name could be: synchronous_mode, failsafe_mode, pause, check_timeline, and so on
-        :returns: `True` if *mode* is enabled in the global configuration.
-        """
-        return bool(parse_bool(self.__config.get(mode)))
-
-    @property
-    def is_paused(self) -> bool:
-        """:returns: `True` if cluster is in maintenance mode."""
-        return self.check_mode('pause')
-
-    @property
-    def is_synchronous_mode(self) -> bool:
-        """:returns: `True` if synchronous replication is requested."""
-        return self.check_mode('synchronous_mode')
-
-    @property
-    def is_synchronous_mode_strict(self) -> bool:
-        """:returns: `True` if at least one synchronous node is required."""
-        return self.check_mode('synchronous_mode_strict')
-
-    def get_standby_cluster_config(self) -> Union[Dict[str, Any], Any]:
-        """:returns: "standby_cluster" configuration."""
-        return deepcopy(self.get('standby_cluster'))
-
-    @property
-    def is_standby_cluster(self) -> bool:
-        """:returns: `True` if global configuration has a valid "standby_cluster" section."""
-        config = self.get_standby_cluster_config()
-        return isinstance(config, dict) and\
-            bool(config.get('host') or config.get('port') or config.get('restore_command'))
-
-    def get_int(self, name: str, default: int = 0) -> int:
-        """Gets current value from the global configuration and trying to return it as int.
-
-        :param name: name of the parameter
-        :param default: default value if *name* is not in the configuration or invalid
-        :returns: currently configured value from the global configuration or *default* if it is not set or invalid.
-        """
-        ret = parse_int(self.get(name))
-        return default if ret is None else ret
-
-    @property
-    def min_synchronous_nodes(self) -> int:
-        """:returns: the minimal number of synchronous nodes based on whether strict mode is requested or not."""
-        return 1 if self.is_synchronous_mode_strict else 0
-
-    @property
-    def synchronous_node_count(self) -> int:
-        """:returns: currently configured value from the global configuration or 1 if it is not set or invalid."""
-        return max(self.get_int('synchronous_node_count', 1), self.min_synchronous_nodes)
-
-    @property
-    def maximum_lag_on_failover(self) -> int:
-        """:returns: currently configured value from the global configuration or 1048576 if it is not set or invalid."""
-        return self.get_int('maximum_lag_on_failover', 1048576)
-
-    @property
-    def maximum_lag_on_syncnode(self) -> int:
-        """:returns: currently configured value from the global configuration or -1 if it is not set or invalid."""
-        return self.get_int('maximum_lag_on_syncnode', -1)
-
-    @property
-    def primary_start_timeout(self) -> int:
-        """:returns: currently configured value from the global configuration or 300 if it is not set or invalid."""
-        default = 300
-        return self.get_int('primary_start_timeout', default)\
-            if 'primary_start_timeout' in self.__config else self.get_int('master_start_timeout', default)
-
-    @property
-    def primary_stop_timeout(self) -> int:
-        """:returns: currently configured value from the global configuration or 300 if it is not set or invalid."""
-        default = 0
-        return self.get_int('primary_stop_timeout', default)\
-            if 'primary_stop_timeout' in self.__config else self.get_int('master_stop_timeout', default)
-
-
-def get_global_config(cluster: Union[Cluster, None], default: Optional[Dict[str, Any]] = None) -> GlobalConfig:
-    """Instantiates :class:`GlobalConfig` based on the input.
-
-    :param cluster: the currently known cluster state from DCS
-    :param default: default configuration, which will be used if there is no valid *cluster.config*
-    :returns: :class:`GlobalConfig` object
-    """
-    # Try to protect from the case when DCS was wiped out
-    if cluster and cluster.config and cluster.config.modify_version:
-        config = cluster.config.data
-    else:
-        config = default or {}
-    return GlobalConfig(deepcopy(config))
-
-
 class Config(object):
-    """
+    """Handle Patroni configuration.
+
     This class is responsible for:
 
-      1) Building and giving access to `effective_configuration` from:
+      1) Building and giving access to ``effective_configuration`` from:
-         * `Config.__DEFAULT_CONFIG` -- some sane default values
-         * `dynamic_configuration` -- configuration stored in DCS
-         * `local_configuration` -- configuration from `config.yml` or environment
 
-      2) Saving and loading `dynamic_configuration` into 'patroni.dynamic.json' file
+         * ``Config.__DEFAULT_CONFIG`` -- some sane default values;
+         * ``dynamic_configuration`` -- configuration stored in DCS;
+         * ``local_configuration`` -- configuration from `config.yml` or environment.
+
+      2) Saving and loading ``dynamic_configuration`` into 'patroni.dynamic.json' file
          located in local_configuration['postgresql']['data_dir'] directory.
-         This is necessary to be able to restore `dynamic_configuration`
+         This is necessary to be able to restore ``dynamic_configuration``
-         if DCS was accidentally wiped
+         if DCS was accidentally wiped.
 
-      3) Loading of configuration file in the old format and converting it into new format
+      3) Loading of configuration file in the old format and converting it into new format.
 
-      4) Mimicking some of the `dict` interfaces to make it possible
+      4) Mimicking some ``dict`` interfaces to make it possible
-         to work with it as with the old `config` object.
+         to work with it as with the old ``config`` object.
+
+    :cvar PATRONI_CONFIG_VARIABLE: name of the environment variable that can be used to load Patroni configuration from.
+    :cvar __CACHE_FILENAME: name of the file used to cache dynamic configuration under Postgres data directory.
+    :cvar __DEFAULT_CONFIG: default configuration values for some Patroni settings.
     """
 
     PATRONI_CONFIG_VARIABLE = PATRONI_ENV_PREFIX + 'CONFIGURATION'
@@ -193,21 +96,38 @@ class Config(object):
             'recovery_min_apply_delay': ''
         },
         'postgresql': {
-            'bin_dir': '',
             'use_slots': True,
             'parameters': CaseInsensitiveDict({p: v[0] for p, v in ConfigHandler.CMDLINE_OPTIONS.items()
-                                               if p not in ('wal_keep_segments', 'wal_keep_size')})
+                                               if v[0] is not None and p not in ('wal_keep_segments', 'wal_keep_size')})
         }
     }
 
     def __init__(self, configfile: str,
                  validator: Optional[Callable[[Dict[str, Any]], List[str]]] = default_validator) -> None:
+        """Create a new instance of :class:`Config` and validate the loaded configuration using *validator*.
+
+        .. note::
+            Patroni will read configuration from these locations in this order:
+
+              * file or directory path passed as command-line argument (*configfile*), if it exists and the file or
+                files found in the directory can be parsed (see :meth:`~Config._load_config_path`), otherwise
+              * YAML file passed via the environment variable (see :attr:`PATRONI_CONFIG_VARIABLE`), if the referenced
+                file exists and can be parsed, otherwise
+              * from configuration values defined as environment variables, see
+                :meth:`~Config._build_environment_configuration`.
+
+        :param configfile: path to Patroni configuration file.
+        :param validator: function used to validate Patroni configuration. It should receive a dictionary which
+            represents Patroni configuration, and return a list of zero or more error messages based on validation.
+
+        :raises:
+            :class:`ConfigParseError`: if any issue is reported by *validator*.
+        """
         self._modify_version = -1
         self._dynamic_configuration = {}
 
         self.__environment_configuration = self._build_environment_configuration()
 
-        # Patroni reads the configuration from the command-line argument if it exists, otherwise from the environment
         self._config_file = configfile if configfile and os.path.exists(configfile) else None
         if self._config_file:
             self._local_configuration = self._load_config_file()
@@ -223,21 +143,49 @@ class Config(object):
         self.__effective_configuration = self._build_effective_configuration({}, self._local_configuration)
         self._data_dir = self.__effective_configuration.get('postgresql', {}).get('data_dir', "")
         self._cache_file = os.path.join(self._data_dir, self.__CACHE_FILENAME)
-        self._load_cache()
+        if validator:  # patronictl uses validator=None
+            self._load_cache()  # we don't want to load anything from local cache for ctl
+            self._validate_failover_tags()  # irrelevant for ctl
         self._cache_needs_saving = False
 
     @property
-    def config_file(self) -> Union[str, None]:
+    def config_file(self) -> Optional[str]:
+        """Path to Patroni configuration file, if any, else ``None``."""
         return self._config_file
 
     @property
     def dynamic_configuration(self) -> Dict[str, Any]:
+        """Deep copy of cached Patroni dynamic configuration."""
         return deepcopy(self._dynamic_configuration)
 
-    def _load_config_path(self, path: str) -> Dict[str, Any]:
+    @property
+    def local_configuration(self) -> Dict[str, Any]:
+        """Deep copy of cached Patroni local configuration.
+
+        :returns: copy of :attr:`~Config._local_configuration`
         """
-        If path is a file, loads the yml file pointed to by path.
+        return deepcopy(dict(self._local_configuration))
-        If path is a directory, loads all yml files in that directory in alphabetical order
+
+    @classmethod
+    def get_default_config(cls) -> Dict[str, Any]:
+        """Deep copy default configuration.
+
+        :returns: copy of :attr:`~Config.__DEFAULT_CONFIG`
+        """
+        return deepcopy(cls.__DEFAULT_CONFIG)
+
+    def _load_config_path(self, path: str) -> Dict[str, Any]:
+        """Load Patroni configuration file(s) from *path*.
+
+        If *path* is a file, load the yml file pointed to by *path*.
+        If *path* is a directory, load all yml files in that directory in alphabetical order.
+
+        :param path: path to either an YAML configuration file, or to a folder containing YAML configuration files.
+
+        :returns: configuration after reading the configuration file(s) from *path*.
+
+        :raises:
+            :class:`ConfigParseError`: if *path* is invalid.
         """
         if os.path.isfile(path):
             files = [path]
@@ -256,14 +204,18 @@ class Config(object):
         return overall_config
 
     def _load_config_file(self) -> Dict[str, Any]:
-        """Loads config.yaml from filesystem and applies some values which were set via ENV"""
+        """Load configuration file(s) from filesystem and apply values which were set via environment variables.
+
+        :returns: final configuration after merging configuration file(s) and environment variables.
+        """
         if TYPE_CHECKING:  # pragma: no cover
-            assert self._config_file is not None
+            assert self.config_file is not None
-        config = self._load_config_path(self._config_file)
+        config = self._load_config_path(self.config_file)
         patch_config(config, self.__environment_configuration)
         return config
 
     def _load_cache(self) -> None:
+        """Load dynamic configuration from ``patroni.dynamic.json``."""
         if os.path.isfile(self._cache_file):
             try:
                 with open(self._cache_file) as f:
@@ -272,14 +224,22 @@ class Config(object):
                 logger.exception('Exception when loading file: %s', self._cache_file)
 
     def save_cache(self) -> None:
+        """Save dynamic configuration to ``patroni.dynamic.json`` under Postgres data directory.
+
+        .. note::
+            ``patroni.dynamic.jsonXXXXXX`` is created as a temporary file and than renamed to ``patroni.dynamic.json``,
+            where ``XXXXXX`` is a random suffix.
+        """
         if self._cache_needs_saving:
             tmpfile = fd = None
             try:
+                pg_perm.set_permissions_from_data_directory(self._data_dir)
                 (fd, tmpfile) = tempfile.mkstemp(prefix=self.__CACHE_FILENAME, dir=self._data_dir)
                 with os.fdopen(fd, 'w') as f:
                     fd = None
                     json.dump(self.dynamic_configuration, f)
                 tmpfile = shutil.move(tmpfile, self._cache_file)
+                os.chmod(self._cache_file, pg_perm.file_create_mode)
                 self._cache_needs_saving = False
             except Exception:
                 logger.exception('Exception when saving file: %s', self._cache_file)
@@ -294,16 +254,84 @@ class Config(object):
                     except Exception:
                         logger.error('Can not remove temporary file %s', tmpfile)
 
+    def __get_and_maybe_adjust_int_value(self, config: Dict[str, Any], param: str, min_value: int) -> int:
+        """Get, validate and maybe adjust a *param* integer value from the *config* :class:`dict`.
+
+        .. note:
+            If the value is smaller than provided *min_value* we update the *config*.
+
+            This method may raise an exception if value isn't :class:`int` or cannot be casted to :class:`int`.
+
+        :param config: :class:`dict` object with new global configuration.
+        :param param: name of the configuration parameter we want to read/validate/adjust.
+        :param min_value: the minimum possible value that a given *param* could have.
+
+        :returns: an integer value which corresponds to a provided *param*.
+        """
+        value = int(config.get(param, self.__DEFAULT_CONFIG[param]))
+        if value < min_value:
+            logger.warning("%s=%d can't be smaller than %d, adjusting...", param, value, min_value)
+            value = config[param] = min_value
+        return value
+
+    def _validate_and_adjust_timeouts(self, config: Dict[str, Any]) -> None:
+        """Validate and adjust ``loop_wait``, ``retry_timeout``, and ``ttl`` values if necessary.
+
+        Minimum values:
+
+            * ``loop_wait``: 1 second;
+            * ``retry_timeout``: 3 seconds.
+            * ``ttl``: 20 seconds;
+
+        Maximum values:
+        In case if values don't fulfill the following rule, ``retry_timeout`` and ``loop_wait``
+        are reduced so that the rule is fulfilled:
+
+            .. code-block:: python
+
+                loop_wait + 2 * retry_timeout <= ttl
+
+        .. note:
+            We prefer to reduce ``loop_wait`` and will reduce ``retry_timeout`` only if ``loop_wait``
+            is already set to a minimal possible value.
+
+        :param config: :class:`dict` object with new global configuration.
+        """
+
+        min_loop_wait = 1
+        loop_wait = self. __get_and_maybe_adjust_int_value(config, 'loop_wait', min_loop_wait)
+        retry_timeout = self. __get_and_maybe_adjust_int_value(config, 'retry_timeout', 3)
+        ttl = self. __get_and_maybe_adjust_int_value(config, 'ttl', 20)
+
+        if min_loop_wait + 2 * retry_timeout > ttl:
+            config['loop_wait'] = min_loop_wait
+            config['retry_timeout'] = (ttl - min_loop_wait) // 2
+            logger.warning('Violated the rule "loop_wait + 2*retry_timeout <= ttl", where ttl=%d. '
+                           'Adjusting loop_wait from %d to %d and retry_timeout from %d to %d',
+                           ttl, loop_wait, min_loop_wait, retry_timeout, config['retry_timeout'])
+        elif loop_wait + 2 * retry_timeout > ttl:
+            config['loop_wait'] = ttl - 2 * retry_timeout
+            logger.warning('Violated the rule "loop_wait + 2*retry_timeout <= ttl", where ttl=%d and retry_timeout=%d.'
+                           ' Adjusting loop_wait from %d to %d', ttl, retry_timeout, loop_wait, config['loop_wait'])
+
     # configuration could be either ClusterConfig or dict
     def set_dynamic_configuration(self, configuration: Union[ClusterConfig, Dict[str, Any]]) -> bool:
+        """Set dynamic configuration values with given *configuration*.
+
+        :param configuration: new dynamic configuration values. Supports :class:`dict` for backward compatibility.
+
+        :returns: ``True`` if changes have been detected between current dynamic configuration and the new dynamic
+            *configuration*, ``False`` otherwise.
+        """
         if isinstance(configuration, ClusterConfig):
             if self._modify_version == configuration.modify_version:
-                return False  # If the version didn't changed there is nothing to do
+                return False  # If the version didn't change there is nothing to do
             self._modify_version = configuration.modify_version
             configuration = configuration.data
 
         if not deep_compare(self._dynamic_configuration, configuration):
             try:
+                self._validate_and_adjust_timeouts(configuration)
                 self.__effective_configuration = self._build_effective_configuration(configuration,
                                                                                      self._local_configuration)
                 self._dynamic_configuration = configuration
@@ -314,6 +342,14 @@ class Config(object):
         return False
 
     def reload_local_configuration(self) -> Optional[bool]:
+        """Reload configuration values from the configuration file(s).
+
+        .. note::
+            Designed to be used when user applies changes to configuration file(s), so Patroni can use the new values
+            with a reload instead of a restart.
+
+        :returns: ``True`` if changes have been detected between current local configuration
+        """
         if self.config_file:
             try:
                 configuration = self._load_config_file()
@@ -321,6 +357,7 @@ class Config(object):
                     new_configuration = self._build_effective_configuration(self._dynamic_configuration, configuration)
                     self._local_configuration = configuration
                     self.__effective_configuration = new_configuration
+                    self._validate_failover_tags()
                     return True
                 else:
                     logger.info('No local configuration items changed.')
@@ -329,12 +366,82 @@ class Config(object):
 
     @staticmethod
     def _process_postgresql_parameters(parameters: Dict[str, Any], is_local: bool = False) -> Dict[str, Any]:
-        return {name: value for name, value in (parameters or {}).items()
+        """Process Postgres *parameters*.
-                if name not in ConfigHandler.CMDLINE_OPTIONS
+
-                or not is_local and ConfigHandler.CMDLINE_OPTIONS[name][1](value)}
+        .. note::
+            If *is_local* configuration discard any setting from *parameters* that is listed under
+            :attr:`~patroni.postgresql.config.ConfigHandler.CMDLINE_OPTIONS` as those are supposed to be set only
+            through dynamic configuration.
+
+            When setting parameters from :attr:`~patroni.postgresql.config.ConfigHandler.CMDLINE_OPTIONS` through
+            dynamic configuration their value will be validated as per the validator defined in that very same
+            attribute entry. If the given value cannot be validated, a warning will be logged and the default value of
+            the GUC will be used instead.
+
+            Some parameters from :attr:`~patroni.postgresql.config.ConfigHandler.CMDLINE_OPTIONS` cannot be set even if
+            not *is_local* configuration:
+
+                * ``listen_addresses``: inferred from ``postgresql.listen`` local configuration or from
+                    ``PATRONI_POSTGRESQL_LISTEN`` environment variable;
+                * ``port``: inferred from ``postgresql.listen`` local configuration or from
+                    ``PATRONI_POSTGRESQL_LISTEN`` environment variable;
+                * ``cluster_name``: set through ``scope`` local configuration or through ``PATRONI_SCOPE`` environment
+                    variable;
+                * ``hot_standby``: always enabled;
+                * ``wal_log_hints``: always enabled.
+
+        :param parameters: Postgres parameters to be processed. Should be the parsed YAML value of
+            ``postgresql.parameters`` configuration, either from local or from dynamic configuration.
+
+        :param is_local: should be ``True`` if *parameters* refers to local configuration, or ``False`` if *parameters*
+            refers to dynamic configuration.
+
+        :returns: new value for ``postgresql.parameters`` after processing and validating *parameters*.
+        """
+        pg_params: Dict[str, Any] = {}
+
+        for name, value in (parameters or {}).items():
+            if name not in ConfigHandler.CMDLINE_OPTIONS:
+                pg_params[name] = value
+            elif not is_local:
+                validator = ConfigHandler.CMDLINE_OPTIONS[name][1]
+                if validator(value):
+                    int_val = parse_int(value) if isinstance(validator, IntValidator) else None
+                    pg_params[name] = int_val if isinstance(int_val, int) else value
+                else:
+                    logger.warning("postgresql parameter %s=%s failed validation, defaulting to %s",
+                                   name, value, ConfigHandler.CMDLINE_OPTIONS[name][0])
+
+        return pg_params
 
     def _safe_copy_dynamic_configuration(self, dynamic_configuration: Dict[str, Any]) -> Dict[str, Any]:
-        config = deepcopy(self.__DEFAULT_CONFIG)
+        """Create a copy of *dynamic_configuration*.
+
+        Merge *dynamic_configuration* with :attr:`__DEFAULT_CONFIG` (*dynamic_configuration* takes precedence), and
+        process ``postgresql.parameters`` from *dynamic_configuration* through :func:`_process_postgresql_parameters`,
+        if present.
+
+        .. note::
+            The following settings are not allowed in ``postgresql`` section as they are intended to be local
+            configuration, and are removed if present:
+
+                * ``connect_address``;
+                * ``proxy_address``;
+                * ``listen``;
+                * ``config_dir``;
+                * ``data_dir``;
+                * ``pgpass``;
+                * ``authentication``;
+
+            Besides that any setting present in *dynamic_configuration* but absent from :attr:`__DEFAULT_CONFIG` is
+            discarded.
+
+        :param dynamic_configuration: Patroni dynamic configuration.
+
+        :returns: copy of *dynamic_configuration*, merged with default dynamic configuration and with some sanity checks
+            performed over it.
+        """
+        config = self.get_default_config()
 
         for name, value in dynamic_configuration.items():
             if name == 'postgresql':
@@ -354,9 +461,25 @@ class Config(object):
 
     @staticmethod
     def _build_environment_configuration() -> Dict[str, Any]:
+        """Get local configuration settings that were specified through environment variables.
+
+        :returns: dictionary containing the found environment variables and their values, respecting the expected
+            structure of Patroni configuration.
+        """
         ret: Dict[str, Any] = defaultdict(dict)
 
-        def _popenv(name: str) -> Union[str, None]:
+        def _popenv(name: str) -> Optional[str]:
+            """Get value of environment variable *name*.
+
+            .. note::
+                *name* is prefixed with :data:`~patroni.PATRONI_ENV_PREFIX` when searching in the environment.
+
+                Also, the corresponding environment variable is removed from the environment upon reading its value.
+
+            :param name: name of the environment variable.
+
+            :returns: value of *name*, if present in the environment, otherwise ``None``.
+            """
             return os.environ.pop(PATRONI_ENV_PREFIX + name.upper(), None)
 
         for param in ('name', 'namespace', 'scope'):
@@ -365,6 +488,23 @@ class Config(object):
                 ret[param] = value
 
         def _fix_log_env(name: str, oldname: str) -> None:
+            """Normalize a log related environment variable.
+
+            .. note::
+                Patroni used to support different names for log related environment variables in the past. As the
+                environment variables were renamed, this function takes care of mapping and normalizing the environment.
+
+                *name* is prefixed with :data:`~patroni.PATRONI_ENV_PREFIX` and ``LOG`` when searching in the
+                environment.
+
+                *oldname* is prefixed with :data:`~patroni.PATRONI_ENV_PREFIX` when searching in the environment.
+
+                If both *name* and *oldname* are set in the environment, *name* takes precedence.
+
+            :param name: new name of a log related environment variable.
+            :param oldname: original name of a log related environment variable.
+            :type oldname: str
+            """
             value = _popenv(oldname)
             name = PATRONI_ENV_PREFIX + 'LOG_' + name.upper()
             if value and name not in os.environ:
@@ -374,6 +514,15 @@ class Config(object):
             _fix_log_env(name, oldname)
 
         def _set_section_values(section: str, params: List[str]) -> None:
+            """Get value of *params* environment variables that are related with *section*.
+
+            .. note::
+                The values are retrieved from the environment and updated directly into the returning dictionary of
+                :func:`_build_environment_configuration`.
+
+            :param section: configuration section the *params* belong to.
+            :param params: name of the Patroni settings.
+            """
             for param in params:
                 value = _popenv(section + '_' + param)
                 if value:
@@ -386,8 +535,8 @@ class Config(object):
         _set_section_values('ctl', ['insecure', 'cacert', 'certfile', 'keyfile', 'keyfile_password'])
         _set_section_values('postgresql', ['listen', 'connect_address', 'proxy_address',
                                            'config_dir', 'data_dir', 'pgpass', 'bin_dir'])
-        _set_section_values('log', ['level', 'traceback_level', 'format', 'dateformat', 'max_queue_size',
+        _set_section_values('log', ['type', 'level', 'traceback_level', 'format', 'dateformat', 'static_fields',
-                                    'dir', 'file_size', 'file_num', 'loggers'])
+                                    'max_queue_size', 'dir', 'file_size', 'file_num', 'loggers'])
         _set_section_values('raft', ['data_dir', 'self_addr', 'partner_addrs', 'password', 'bind_addr'])
 
         for binary in ('pg_ctl', 'initdb', 'pg_controldata', 'pg_basebackup', 'postgres', 'pg_isready', 'pg_rewind'):
@@ -395,6 +544,7 @@ class Config(object):
             if value:
                 ret['postgresql'].setdefault('bin_name', {})[binary] = value
 
+        # parse all values retrieved from the environment as Python objects, according to the expected type
         for first, second in (('restapi', 'allowlist_include_members'), ('ctl', 'insecure')):
             value = ret.get(first, {}).pop(second, None)
             if value:
@@ -411,7 +561,13 @@ class Config(object):
                     if value is not None:
                         ret[first][second] = value
 
-        def _parse_list(value: str) -> Union[List[str], None]:
+        def _parse_list(value: str) -> Optional[List[str]]:
+            """Parse an YAML list *value* as a :class:`list`.
+
+            :param value: YAML list as a string.
+
+            :returns: *value* as :class:`list`.
+            """
             if not (value.strip().startswith('-') or '[' in value):
                 value = '[{0}]'.format(value)
             try:
@@ -427,7 +583,19 @@ class Config(object):
                 if value:
                     ret[first][second] = value
 
-        def _parse_dict(value: str) -> Union[Dict[str, Any], None]:
+        logformat = ret.get('log', {}).get('format')
+        if logformat and not re.search(r'%\(\w+\)', logformat):
+            logformat = _parse_list(logformat)
+            if logformat:
+                ret['log']['format'] = logformat
+
+        def _parse_dict(value: str) -> Optional[Dict[str, Any]]:
+            """Parse an YAML dictionary *value* as a :class:`dict`.
+
+            :param value: YAML dictionary as a string.
+
+            :returns: *value* as :class:`dict`.
+            """
             if not value.strip().startswith('{'):
                 value = '{{{0}}}'.format(value)
             try:
@@ -436,7 +604,12 @@ class Config(object):
                 logger.exception('Exception when parsing dict %s', value)
                 return None
 
-        for first, params in (('restapi', ('http_extra_headers', 'https_extra_headers')), ('log', ('loggers',))):
+        dict_configs = (
+            ('restapi', ('http_extra_headers', 'https_extra_headers')),
+            ('log', ('static_fields', 'loggers'))
+        )
+
+        for first, params in dict_configs:
             for second in params:
                 value = ret.get(first, {}).pop(second, None)
                 if value:
@@ -444,17 +617,25 @@ class Config(object):
                     if value:
                         ret[first][second] = value
 
-        def _get_auth(name: str, params: Optional[Collection[str]] = None) -> Dict[str, str]:
+        def _get_auth(name: str, params: Collection[str] = _AUTH_ALLOWED_PARAMETERS[:2]) -> Dict[str, str]:
+            """Get authorization related environment variables *params* from section *name*.
+
+            :param name: name of a configuration section that may contain authorization *params*.
+            :param params: the authorization settings that may be set under section *name*.
+
+            :returns: dictionary containing environment values for authorization *params* of section *name*.
+            """
             ret: Dict[str, str] = {}
-            for param in params or _AUTH_ALLOWED_PARAMETERS[:2]:
+            for param in params:
                 value = _popenv(name + '_' + param)
                 if value:
                     ret[param] = value
             return ret
 
-        restapi_auth = _get_auth('restapi')
+        for section in ('ctl', 'restapi'):
-        if restapi_auth:
+            auth = _get_auth(section)
-            ret['restapi']['authentication'] = restapi_auth
+            if auth:
+                ret[section]['authentication'] = auth
 
         authentication = {}
         for user_type in ('replication', 'superuser', 'rewind'):
@@ -468,13 +649,14 @@ class Config(object):
         for param in list(os.environ.keys()):
             if param.startswith(PATRONI_ENV_PREFIX):
                 # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
-                name, suffix = (param[8:].split('_', 1) + [''])[:2]
+                name, suffix = (param[len(PATRONI_ENV_PREFIX):].split('_', 1) + [''])[:2]
                 if suffix in ('HOST', 'HOSTS', 'PORT', 'USE_PROXIES', 'PROTOCOL', 'SRV', 'SRV_SUFFIX', 'URL', 'PROXY',
                               'CACERT', 'CERT', 'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC', 'CONSISTENCY',
                               'REGISTER_SERVICE', 'SERVICE_CHECK_INTERVAL', 'SERVICE_CHECK_TLS_SERVER_NAME',
                               'SERVICE_TAGS', 'NAMESPACE', 'CONTEXT', 'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL',
                               'POD_IP', 'PORTS', 'LABELS', 'BYPASS_API_SERVICE', 'RETRIABLE_HTTP_CODES', 'KEY_PASSWORD',
-                              'USE_SSL', 'SET_ACLS', 'GROUP', 'DATABASE') and name:
+                              'USE_SSL', 'SET_ACLS', 'GROUP', 'DATABASE', 'LEADER_LABEL_VALUE', 'FOLLOWER_LABEL_VALUE',
+                              'STANDBY_LEADER_LABEL_VALUE', 'TMP_ROLE_LABEL', 'AUTH_DATA') and name:
                     value = os.environ.pop(param)
                     if name == 'CITUS':
                         if suffix == 'GROUP':
@@ -485,7 +667,7 @@ class Config(object):
                         value = value and parse_int(value)
                     elif suffix in ('HOSTS', 'PORTS', 'CHECKS', 'SERVICE_TAGS', 'RETRIABLE_HTTP_CODES'):
                         value = value and _parse_list(value)
-                    elif suffix in ('LABELS', 'SET_ACLS'):
+                    elif suffix in ('LABELS', 'SET_ACLS', 'AUTH_DATA'):
                         value = _parse_dict(value)
                     elif suffix in ('USE_PROXIES', 'REGISTER_SERVICE', 'USE_ENDPOINTS', 'BYPASS_API_SERVICE', 'VERIFY'):
                         value = parse_bool(value)
@@ -498,14 +680,14 @@ class Config(object):
         users = {}
         for param in list(os.environ.keys()):
             if param.startswith(PATRONI_ENV_PREFIX):
-                name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
+                name, suffix = (param[len(PATRONI_ENV_PREFIX):].rsplit('_', 1) + [''])[:2]
                 # PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>
                 # CREATE USER "<username>" WITH <OPTIONS> PASSWORD '<password>'
                 if name and suffix == 'PASSWORD':
                     password = os.environ.pop(param)
                     if password:
                         users[name] = {'password': password}
-                        options = os.environ.pop(param[:-9] + '_OPTIONS', None)
+                        options = os.environ.pop(param[:-9] + '_OPTIONS', None)  # replace "_PASSWORD" with "_OPTIONS"
                         options = options and _parse_list(options)
                         if options:
                             users[name]['options'] = options
@@ -516,6 +698,16 @@ class Config(object):
 
     def _build_effective_configuration(self, dynamic_configuration: Dict[str, Any],
                                        local_configuration: Dict[str, Union[Dict[str, Any], Any]]) -> Dict[str, Any]:
+        """Build effective configuration by merging *dynamic_configuration* and *local_configuration*.
+
+        .. note::
+            *local_configuration* takes precedence over *dynamic_configuration* if a setting is defined in both.
+
+        :param dynamic_configuration: Patroni dynamic configuration.
+        :param local_configuration: Patroni local configuration.
+
+        :returns: _description_
+        """
         config = self._safe_copy_dynamic_configuration(dynamic_configuration)
         for name, value in local_configuration.items():
             if name == 'citus':  # remove invalid citus configuration
@@ -531,9 +723,10 @@ class Config(object):
             elif name not in config or name in ['watchdog']:
                 config[name] = deepcopy(value) if value else {}
 
-        # restapi server expects to get restapi.auth = 'username:password'
+        # restapi server expects to get restapi.auth = 'username:password' and similarly for `ctl`
-        if 'restapi' in config and 'authentication' in config['restapi']:
+        for section in ('ctl', 'restapi'):
-            config['restapi']['auth'] = '{username}:{password}'.format(**config['restapi']['authentication'])
+            if section in config and 'authentication' in config[section]:
+                config[section]['auth'] = '{username}:{password}'.format(**config[section]['authentication'])
 
         # special treatment for old config
 
@@ -564,7 +757,7 @@ class Config(object):
         if 'citus' in config:
             bootstrap = config.setdefault('bootstrap', {})
             dcs = bootstrap.setdefault('dcs', {})
-            dcs.setdefault('synchronous_mode', True)
+            dcs.setdefault('synchronous_mode', 'quorum')
 
         updated_fields = (
             'name',
@@ -578,23 +771,68 @@ class Config(object):
         return config
 
     def get(self, key: str, default: Optional[Any] = None) -> Any:
+        """Get effective value of ``key`` setting from Patroni configuration root.
+
+        Designed to work the same way as :func:`dict.get`.
+
+        :param key: name of the setting.
+        :param default: default value if *key* is not present in the effective configuration.
+
+        :returns: value of *key*, if present in the effective configuration, otherwise *default*.
+        """
         return self.__effective_configuration.get(key, default)
 
     def __contains__(self, key: str) -> bool:
+        """Check if setting *key* is present in the effective configuration.
+
+        Designed to work the same way as :func:`dict.__contains__`.
+
+        :param key: name of the setting to be checked.
+
+        :returns: ``True`` if setting *key* exists in effective configuration, else ``False``.
+        """
         return key in self.__effective_configuration
 
     def __getitem__(self, key: str) -> Any:
+        """Get value of setting *key* from effective configuration.
+
+        Designed to work the same way as :func:`dict.__getitem__`.
+
+        :param key: name of the setting.
+
+        :returns: value of setting *key*.
+
+        :raises:
+            :class:`KeyError`: if *key* is not present in effective configuration.
+        """
         return self.__effective_configuration[key]
 
     def copy(self) -> Dict[str, Any]:
+        """Get a deep copy of effective Patroni configuration.
+
+        :returns: a deep copy of the Patroni configuration.
+        """
         return deepcopy(self.__effective_configuration)
 
-    def get_global_config(self, cluster: Union[Cluster, None]) -> GlobalConfig:
+    def _validate_failover_tags(self) -> None:
-        """Instantiate :class:`GlobalConfig` based on input.
+        """Check ``nofailover``/``failover_priority`` config and warn user if it's contradictory.
 
-        Use the configuration from provided *cluster* (the most up-to-date) or from the
+        .. note::
-        local cache if *cluster.config* is not initialized or doesn't have a valid config.
+          To preserve sanity (and backwards compatibility) the ``nofailover`` tag will still exist. A contradictory
-        :param cluster: the currently known cluster state from DCS
+          configuration is one where ``nofailover`` is ``True`` but ``failover_priority > 0``, or where
-        :returns: :class:`GlobalConfig` object
+          ``nofailover`` is ``False``, but ``failover_priority <= 0``. Essentially, ``nofailover`` and
+          ``failover_priority`` are communicating different things.
+          This checks for this edge case (which is a misconfiguration on the part of the user) and warns them.
+          The behaviour is as if ``failover_priority`` were not provided (i.e ``nofailover`` is the
+          bedrock source of truth)
         """
-        return get_global_config(cluster, self._dynamic_configuration)
+        tags = self.get('tags', {})
+        if 'nofailover' not in tags:
+            return
+        nofailover_tag = tags.get('nofailover')
+        failover_priority_tag = parse_int(tags.get('failover_priority'))
+        if failover_priority_tag is not None \
+                and (bool(nofailover_tag) is True and failover_priority_tag > 0
+                     or bool(nofailover_tag) is False and failover_priority_tag <= 0):
+            logger.warning('Conflicting configuration between nofailover: %s and failover_priority: %s. '
+                           'Defaulting to nofailover: %s', nofailover_tag, failover_priority_tag, nofailover_tag)

patroni/config_generator.py

@@ -0,0 +1,512 @@
+"""patroni ``--generate-config`` machinery."""
+import abc
+import logging
+import os
+import psutil
+import socket
+import sys
+import yaml
+
+from getpass import getuser, getpass
+from contextlib import contextmanager
+from typing import Any, Dict, Iterator, List, Optional, TextIO, Tuple, TYPE_CHECKING, Union
+if TYPE_CHECKING:  # pragma: no cover
+    from psycopg import Cursor
+    from psycopg2 import cursor
+
+from . import psycopg
+from .config import Config
+from .exceptions import PatroniException
+from .log import PatroniLogger
+from .postgresql.config import ConfigHandler, parse_dsn
+from .postgresql.misc import postgres_major_version_to_int
+from .utils import get_major_version, parse_bool, patch_config, read_stripped
+
+
+# Mapping between the libpq connection parameters and the environment variables.
+# This dict should be kept in sync with `patroni.utils._AUTH_ALLOWED_PARAMETERS`
+# (we use "username" in the Patroni config for some reason, other parameter names are the same).
+_AUTH_ALLOWED_PARAMETERS_MAPPING = {
+    'user': 'PGUSER',
+    'password': 'PGPASSWORD',
+    'sslmode': 'PGSSLMODE',
+    'sslcert': 'PGSSLCERT',
+    'sslkey': 'PGSSLKEY',
+    'sslpassword': '',
+    'sslrootcert': 'PGSSLROOTCERT',
+    'sslcrl': 'PGSSLCRL',
+    'sslcrldir': 'PGSSLCRLDIR',
+    'gssencmode': 'PGGSSENCMODE',
+    'channel_binding': 'PGCHANNELBINDING'
+}
+NO_VALUE_MSG = '#FIXME'
+
+
+def get_address() -> Tuple[str, str]:
+    """Try to get hostname and the ip address for it returned by :func:`~socket.gethostname`.
+
+    .. note::
+        Can also return local ip.
+
+    :returns: tuple consisting of the hostname returned by :func:`~socket.gethostname`
+        and the first element in the sorted list of the addresses returned by :func:`~socket.getaddrinfo`.
+        Sorting guarantees it will prefer IPv4.
+        If an exception occured, hostname and ip values are equal to :data:`~patroni.config_generator.NO_VALUE_MSG`.
+    """
+    hostname = None
+    try:
+        hostname = socket.gethostname()
+        return hostname, sorted(socket.getaddrinfo(hostname, 0, socket.AF_UNSPEC, socket.SOCK_STREAM, 0),
+                                key=lambda x: x[0])[0][4][0]
+    except Exception as err:
+        logging.warning('Failed to obtain address: %r', err)
+        return NO_VALUE_MSG, NO_VALUE_MSG
+
+
+class AbstractConfigGenerator(abc.ABC):
+    """Object representing the generated Patroni config.
+
+    :ivar output_file: full path to the output file to be used.
+    :ivar pg_major: integer representation of the major PostgreSQL version.
+    :ivar config: dictionary used for the generated configuration storage.
+    """
+
+    _HOSTNAME, _IP = get_address()
+
+    def __init__(self, output_file: Optional[str]) -> None:
+        """Set up the output file (if passed), helper vars and the minimal config structure.
+
+        :param output_file: full path to the output file to be used.
+        """
+        self.output_file = output_file
+        self.pg_major = 0
+        self.config = self.get_template_config()
+
+        self.generate()
+
+    @classmethod
+    def get_template_config(cls) -> Dict[str, Any]:
+        """Generate a template config for further extension (e.g. in the inherited classes).
+
+        :returns: dictionary with the values gathered from Patroni env, hopefully defined hostname and ip address
+                  (otherwise set to :data:`~patroni.config_generator.NO_VALUE_MSG`), and some sane defaults.
+        """
+        template_config: Dict[str, Any] = {
+            'scope': NO_VALUE_MSG,
+            'name': cls._HOSTNAME,
+            'restapi': {
+                'connect_address': cls._IP + ':8008',
+                'listen': cls._IP + ':8008'
+            },
+            'log': {
+                'type': PatroniLogger.DEFAULT_TYPE,
+                'level': PatroniLogger.DEFAULT_LEVEL,
+                'traceback_level': PatroniLogger.DEFAULT_TRACEBACK_LEVEL,
+                'format': PatroniLogger.DEFAULT_FORMAT,
+                'max_queue_size': PatroniLogger.DEFAULT_MAX_QUEUE_SIZE
+            },
+            'postgresql': {
+                'data_dir': NO_VALUE_MSG,
+                'connect_address': cls._IP + ':5432',
+                'listen': cls._IP + ':5432',
+                'bin_dir': '',
+                'authentication': {
+                    'superuser': {
+                        'username': 'postgres',
+                        'password': NO_VALUE_MSG
+                    },
+                    'replication': {
+                        'username': 'replicator',
+                        'password': NO_VALUE_MSG
+                    }
+                }
+            },
+            'tags': {
+                'failover_priority': 1,
+                'noloadbalance': False,
+                'clonefrom': True,
+                'nosync': False,
+                'nostream': False,
+            }
+        }
+
+        dynamic_config = Config.get_default_config()
+        # to properly dump CaseInsensitiveDict as YAML later
+        dynamic_config['postgresql']['parameters'] = dict(dynamic_config['postgresql']['parameters'])
+        config = Config('', None).local_configuration  # Get values from env
+        config.setdefault('bootstrap', {})['dcs'] = dynamic_config
+        config.setdefault('postgresql', {})
+        del config['bootstrap']['dcs']['standby_cluster']
+
+        patch_config(template_config, config)
+        return template_config
+
+    @abc.abstractmethod
+    def generate(self) -> None:
+        """Generate config and store in :attr:`~AbstractConfigGenerator.config`."""
+
+    @staticmethod
+    def _format_block(block: Any, line_prefix: str = '') -> str:
+        """Format a single YAML block.
+
+        .. note::
+            Optionally the formatted block could be indented with the *line_prefix*
+
+        :param block: the object that should be formatted to YAML.
+        :param line_prefix: is used for indentation.
+
+        :returns: a formatted and indented *block*.
+        """
+        return line_prefix + yaml.safe_dump(block, default_flow_style=False, line_break='\n',
+                                            allow_unicode=True, indent=2).strip().replace('\n', '\n' + line_prefix)
+
+    def _format_config_section(self, section_name: str) -> Iterator[str]:
+        """Format and yield as single section of the current :attr:`~AbstractConfigGenerator.config`.
+
+        .. note::
+            If the section is a :class:`dict` object we put an empty line before it.
+
+        :param section_name: a section name in the :attr:`~AbstractConfigGenerator.config`.
+
+        :yields: a formatted section in case if it exists in the :attr:`~AbstractConfigGenerator.config`.
+        """
+        if section_name in self.config:
+            if isinstance(self.config[section_name], dict):
+                yield ''
+            yield self._format_block({section_name: self.config[section_name]})
+
+    def _format_config(self) -> Iterator[str]:
+        """Format current :attr:`~AbstractConfigGenerator.config` and enrich it with some comments.
+
+        :yields: formatted lines or blocks that represent a text output of the YAML document.
+        """
+        for name in ('scope', 'namespace', 'name', 'log', 'restapi', 'ctl', 'citus',
+                     'consul', 'etcd', 'etcd3', 'exhibitor', 'kubernetes', 'raft', 'zookeeper'):
+            yield from self._format_config_section(name)
+
+        if 'bootstrap' in self.config:
+            yield '\n# The bootstrap configuration. Works only when the cluster is not yet initialized.'
+            yield '# If the cluster is already initialized, all changes in the `bootstrap` section are ignored!'
+            yield 'bootstrap:'
+            if 'dcs' in self.config['bootstrap']:
+                yield '  # This section will be written into <dcs>:/<namespace>/<scope>/config after initializing'
+                yield '  # new cluster and all other cluster members will use it as a `global configuration`.'
+                yield '  # WARNING! If you want to change any of the parameters that were set up'
+                yield '  # via `bootstrap.dcs` section, please use `patronictl edit-config`!'
+                yield '  dcs:'
+                for name in ('loop_wait', 'retry_timeout', 'ttl'):
+                    if name in self.config['bootstrap']['dcs']:
+                        yield self._format_block({name: self.config['bootstrap']['dcs'].pop(name)}, '    ')
+
+                for name, value in self.config['bootstrap']['dcs'].items():
+                    yield self._format_block({name: value}, '    ')
+
+        for name in ('postgresql', 'watchdog', 'tags'):
+            yield from self._format_config_section(name)
+
+    def _write_config_to_fd(self, fd: TextIO) -> None:
+        """Format and write current :attr:`~AbstractConfigGenerator.config` to provided file descriptor.
+
+        :param fd: where to write the config file. Could be ``sys.stdout`` or the real file.
+        """
+        fd.write('\n'.join(self._format_config()))
+
+    def write_config(self) -> None:
+        """Write current :attr:`~AbstractConfigGenerator.config` to the output file if provided, to stdout otherwise."""
+        if self.output_file:
+            dir_path = os.path.dirname(self.output_file)
+            if dir_path and not os.path.isdir(dir_path):
+                os.makedirs(dir_path)
+            with open(self.output_file, 'w', encoding='UTF-8') as output_file:
+                self._write_config_to_fd(output_file)
+        else:
+            self._write_config_to_fd(sys.stdout)
+
+
+class SampleConfigGenerator(AbstractConfigGenerator):
+    """Object representing the generated sample Patroni config.
+
+    Sane defults are used based on the gathered PG version.
+    """
+
+    @property
+    def get_auth_method(self) -> str:
+        """Return the preferred authentication method for a specific PG version if provided or the default ``md5``.
+
+        :returns: :class:`str` value for the preferred authentication method.
+        """
+        return 'scram-sha-256' if self.pg_major and self.pg_major >= 100000 else 'md5'
+
+    def _get_int_major_version(self) -> int:
+        """Get major PostgreSQL version from the binary as an integer.
+
+        :returns: an integer PostgreSQL major version representation gathered from the PostgreSQL binary.
+                  See :func:`~patroni.postgresql.misc.postgres_major_version_to_int` and
+                  :func:`~patroni.utils.get_major_version`.
+        """
+        postgres_bin = ((self.config.get('postgresql') or {}).get('bin_name') or {}).get('postgres', 'postgres')
+        return postgres_major_version_to_int(get_major_version(self.config['postgresql'].get('bin_dir'), postgres_bin))
+
+    def generate(self) -> None:
+        """Generate sample config using some sane defaults and update :attr:`~AbstractConfigGenerator.config`."""
+        self.pg_major = self._get_int_major_version()
+
+        self.config['postgresql']['parameters'] = {'password_encryption': self.get_auth_method}
+        username = self.config["postgresql"]["authentication"]["replication"]["username"]
+        self.config['postgresql']['pg_hba'] = [
+            f'host all all all {self.get_auth_method}',
+            f'host replication {username} all {self.get_auth_method}'
+        ]
+
+        # add version-specific configuration
+        wal_keep_param = 'wal_keep_segments' if self.pg_major < 130000 else 'wal_keep_size'
+        self.config['bootstrap']['dcs']['postgresql']['parameters'][wal_keep_param] = \
+            ConfigHandler.CMDLINE_OPTIONS[wal_keep_param][0]
+
+        wal_level = 'hot_standby' if self.pg_major < 90600 else 'replica'
+        self.config['bootstrap']['dcs']['postgresql']['parameters']['wal_level'] = wal_level
+
+        self.config['bootstrap']['dcs']['postgresql']['use_pg_rewind'] = True
+        if self.pg_major >= 110000:
+            self.config['postgresql']['authentication'].setdefault(
+                'rewind', {'username': 'rewind_user'}).setdefault('password', NO_VALUE_MSG)
+
+
+class RunningClusterConfigGenerator(AbstractConfigGenerator):
+    """Object representing the Patroni config generated using information gathered from the running instance.
+
+    :ivar dsn: DSN string for the local instance to get GUC values from (if provided).
+    :ivar parsed_dsn: DSN string parsed into a dictionary (see :func:`~patroni.postgresql.config.parse_dsn`).
+    """
+
+    def __init__(self, output_file: Optional[str] = None, dsn: Optional[str] = None) -> None:
+        """Additionally store the passed dsn (if any) in both original and parsed version and run config generation.
+
+        :param output_file: full path to the output file to be used.
+        :param dsn: DSN string for the local instance to get GUC values from.
+
+        :raises:
+            :exc:`~patroni.exceptions.PatroniException`: if DSN parsing failed.
+        """
+        self.dsn = dsn
+        self.parsed_dsn = {}
+
+        super().__init__(output_file)
+
+    @property
+    def _get_hba_conn_types(self) -> Tuple[str, ...]:
+        """Return the connection types allowed.
+
+        If :attr:`~RunningClusterConfigGenerator.pg_major` is defined, adds additional parameters
+        for PostgreSQL version >=16.
+
+        :returns: tuple of the connection methods allowed.
+        """
+        allowed_types = ('local', 'host', 'hostssl', 'hostnossl', 'hostgssenc', 'hostnogssenc')
+        if self.pg_major and self.pg_major >= 160000:
+            allowed_types += ('include', 'include_if_exists', 'include_dir')
+        return allowed_types
+
+    @property
+    def _required_pg_params(self) -> List[str]:
+        """PG configuration prameters that have to be always present in the generated config.
+
+        :returns: list of the parameter names.
+        """
+        return ['hba_file', 'ident_file', 'config_file', 'data_directory'] + \
+            list(ConfigHandler.CMDLINE_OPTIONS.keys())
+
+    def _get_bin_dir_from_running_instance(self) -> str:
+        """Define the directory postgres binaries reside using postmaster's pid executable.
+
+        :returns: path to the PostgreSQL binaries directory.
+
+        :raises:
+            :exc:`~patroni.exceptions.PatroniException`: if:
+
+                * pid could not be obtained from the ``postmaster.pid`` file; or
+                * :exc:`OSError` occured during ``postmaster.pid`` file handling; or
+                * the obtained postmaster pid doesn't exist.
+        """
+        postmaster_pid = None
+        data_dir = self.config['postgresql']['data_dir']
+        try:
+            with open(f"{data_dir}/postmaster.pid", 'r') as pid_file:
+                postmaster_pid = pid_file.readline()
+                if not postmaster_pid:
+                    raise PatroniException('Failed to obtain postmaster pid from postmaster.pid file')
+                postmaster_pid = int(postmaster_pid.strip())
+        except OSError as err:
+            raise PatroniException(f'Error while reading postmaster.pid file: {err}')
+        try:
+            return os.path.dirname(psutil.Process(postmaster_pid).exe())
+        except psutil.NoSuchProcess:
+            raise PatroniException("Obtained postmaster pid doesn't exist.")
+
+    @contextmanager
+    def _get_connection_cursor(self) -> Iterator[Union['cursor', 'Cursor[Any]']]:
+        """Get cursor for the PG connection established based on the stored information.
+
+        :raises:
+            :exc:`~patroni.exceptions.PatroniException`: if :exc:`psycopg.Error` occured.
+        """
+        try:
+            conn = psycopg.connect(dsn=self.dsn,
+                                   password=self.config['postgresql']['authentication']['superuser']['password'])
+            with conn.cursor() as cur:
+                yield cur
+            conn.close()
+        except psycopg.Error as e:
+            raise PatroniException(f'Failed to establish PostgreSQL connection: {e}')
+
+    def _set_pg_params(self, cur: Union['cursor', 'Cursor[Any]']) -> None:
+        """Extend :attr:`~RunningClusterConfigGenerator.config` with the actual PG GUCs values.
+
+        THe following GUC values are set:
+
+            * Non-internal having configuration file, postmaster command line or environment variable
+              as a source.
+
+            * List of the always required parameters (see :meth:`~RunningClusterConfigGenerator._required_pg_params`).
+
+        :param cur: connection cursor to use.
+        """
+        cur.execute("SELECT name, pg_catalog.current_setting(name) FROM pg_catalog.pg_settings "
+                    "WHERE context <> 'internal' "
+                    "AND source IN ('configuration file', 'command line', 'environment variable') "
+                    "AND category <> 'Write-Ahead Log / Recovery Target' "
+                    "AND setting <> '(disabled)' "
+                    "OR name = ANY(%s)", (self._required_pg_params,))
+
+        helper_dict = dict.fromkeys(['port', 'listen_addresses'])
+        self.config['postgresql'].setdefault('parameters', {})
+        for param, value in cur.fetchall():
+            if param == 'data_directory':
+                self.config['postgresql']['data_dir'] = value
+            elif param == 'cluster_name' and value:
+                self.config['scope'] = value
+            elif param in ('archive_command', 'restore_command',
+                           'archive_cleanup_command', 'recovery_end_command',
+                           'ssl_passphrase_command', 'hba_file',
+                           'ident_file', 'config_file'):
+                # write commands to the local config due to security implications
+                # write hba/ident/config_file to local config to ensure they are not removed later
+                self.config['postgresql']['parameters'][param] = value
+            elif param in helper_dict:
+                helper_dict[param] = value
+            else:
+                self.config['bootstrap']['dcs']['postgresql']['parameters'][param] = value
+
+        connect_port = self.parsed_dsn.get('port', os.getenv('PGPORT', helper_dict['port']))
+        self.config['postgresql']['connect_address'] = f'{self._IP}:{connect_port}'
+        self.config['postgresql']['listen'] = f'{helper_dict["listen_addresses"]}:{helper_dict["port"]}'
+
+    def _set_su_params(self) -> None:
+        """Extend :attr:`~RunningClusterConfigGenerator.config` with the superuser auth information.
+
+        Information set is based on the options used for connection.
+        """
+        su_params: Dict[str, str] = {}
+        for conn_param, env_var in _AUTH_ALLOWED_PARAMETERS_MAPPING.items():
+            val = self.parsed_dsn.get(conn_param, os.getenv(env_var))
+            if val:
+                su_params[conn_param] = val
+        patroni_env_su_username = ((self.config.get('authentication') or {}).get('superuser') or {}).get('username')
+        patroni_env_su_pwd = ((self.config.get('authentication') or {}).get('superuser') or {}).get('password')
+        # because we use "username" in the config for some reason
+        su_params['username'] = su_params.pop('user', patroni_env_su_username) or getuser()
+        su_params['password'] = su_params.get('password', patroni_env_su_pwd) or \
+            getpass('Please enter the user password:')
+        self.config['postgresql']['authentication'] = {
+            'superuser': su_params,
+            'replication': {'username': NO_VALUE_MSG, 'password': NO_VALUE_MSG}
+        }
+
+    def _set_conf_files(self) -> None:
+        """Extend :attr:`~RunningClusterConfigGenerator.config` with ``pg_hba.conf`` and ``pg_ident.conf`` content.
+
+        .. note::
+            This function only defines ``postgresql.pg_hba`` and ``postgresql.pg_ident`` when
+            ``hba_file`` and ``ident_file`` are set to the defaults. It may happen these files
+            are located outside of ``PGDATA`` and Patroni doesn't have write permissions for them.
+
+        :raises:
+            :exc:`~patroni.exceptions.PatroniException`: if :exc:`OSError` occured during the conf files handling.
+        """
+        default_hba_path = os.path.join(self.config['postgresql']['data_dir'], 'pg_hba.conf')
+        if self.config['postgresql']['parameters']['hba_file'] == default_hba_path:
+            try:
+                self.config['postgresql']['pg_hba'] = list(
+                    filter(lambda i: i and i.split()[0] in self._get_hba_conn_types, read_stripped(default_hba_path)))
+            except OSError as err:
+                raise PatroniException(f'Failed to read pg_hba.conf: {err}')
+
+        default_ident_path = os.path.join(self.config['postgresql']['data_dir'], 'pg_ident.conf')
+        if self.config['postgresql']['parameters']['ident_file'] == default_ident_path:
+            try:
+                self.config['postgresql']['pg_ident'] = [i for i in read_stripped(default_ident_path)
+                                                         if i and not i.startswith('#')]
+            except OSError as err:
+                raise PatroniException(f'Failed to read pg_ident.conf: {err}')
+            if not self.config['postgresql']['pg_ident']:
+                del self.config['postgresql']['pg_ident']
+
+    def _enrich_config_from_running_instance(self) -> None:
+        """Extend :attr:`~RunningClusterConfigGenerator.config` with the values gathered from the running instance.
+
+        Retrieve the following information from the running PostgreSQL instance:
+
+        * superuser auth parameters (see :meth:`~RunningClusterConfigGenerator._set_su_params`);
+        * some GUC values (see :meth:`~RunningClusterConfigGenerator._set_pg_params`);
+        * ``postgresql.connect_address``, ``postgresql.listen``;
+        * ``postgresql.pg_hba`` and ``postgresql.pg_ident`` (see :meth:`~RunningClusterConfigGenerator._set_conf_files`)
+
+        And redefine ``scope`` with the ``cluster_name`` GUC value if set.
+
+        :raises:
+            :exc:`~patroni.exceptions.PatroniException`: if the provided user doesn't have superuser privileges.
+        """
+        self._set_su_params()
+
+        with self._get_connection_cursor() as cur:
+            self.pg_major = getattr(cur.connection, 'server_version', 0)
+
+            if not parse_bool(cur.connection.info.parameter_status('is_superuser')):
+                raise PatroniException('The provided user does not have superuser privilege')
+
+            self._set_pg_params(cur)
+
+        self._set_conf_files()
+
+    def generate(self) -> None:
+        """Generate config using the info gathered from the specified running PG instance.
+
+        Result is written to :attr:`~RunningClusterConfigGenerator.config`.
+        """
+        if self.dsn:
+            self.parsed_dsn = parse_dsn(self.dsn) or {}
+            if not self.parsed_dsn:
+                raise PatroniException('Failed to parse DSN string')
+
+        self._enrich_config_from_running_instance()
+        self.config['postgresql']['bin_dir'] = self._get_bin_dir_from_running_instance()
+
+
+def generate_config(output_file: str, sample: bool, dsn: Optional[str]) -> None:
+    """Generate Patroni configuration file.
+
+    :param output_file: Full path to the configuration file to be used. If not provided, result is sent to ``stdout``.
+    :param sample: Optional flag. If set, no source instance will be used - generate config with some sane defaults.
+    :param dsn: Optional DSN string for the local instance to get GUC values from.
+    """
+    try:
+        if sample:
+            config_generator = SampleConfigGenerator(output_file)
+        else:
+            config_generator = RunningClusterConfigGenerator(output_file, dsn)
+
+        config_generator.write_config()
+    except PatroniException as e:
+        sys.exit(str(e))
+    except Exception as e:
+        sys.exit(f'Unexpected exception: {e}')

patroni/dcs/consul.py

@@ -15,9 +15,10 @@ from urllib3.exceptions import HTTPError
 from urllib.parse import urlencode, urlparse, quote
 from typing import Any, Callable, Dict, List, Mapping, NamedTuple, Optional, Union, Tuple, TYPE_CHECKING
 
-from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState,\
+from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, Status, SyncState, \
-    TimelineHistory, ReturnFalseException, catch_return_false_exception, citus_group_re
+    TimelineHistory, ReturnFalseException, catch_return_false_exception
 from ..exceptions import DCSError
+from ..postgresql.mpp import AbstractMPP
 from ..utils import deep_compare, parse_bool, Retry, RetryFailedError, split_host_port, uri, USER_AGENT
 if TYPE_CHECKING:  # pragma: no cover
     from ..config import Config
@@ -141,6 +142,36 @@ class HTTPClient(object):
 class ConsulClient(base.Consul):
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """
+        Consul client with Patroni customisations.
+
+        .. note::
+
+            Parameters, *token*, *cert* and *ca_cert* are not passed to the parent class :class:`consul.base.Consul`.
+
+        Original class documentation,
+
+            *token* is an optional ``ACL token``. If supplied it will be used by
+            default for all requests made with this client session. It's still
+            possible to override this token by passing a token explicitly for a
+            request.
+
+            *consistency* sets the consistency mode to use by default for all reads
+            that support the consistency option. It's still possible to override
+            this by passing explicitly for a given request. *consistency* can be
+            either 'default', 'consistent' or 'stale'.
+
+            *dc* is the datacenter that this agent will communicate with.
+            By default, the datacenter of the host is used.
+
+            *verify* is whether to verify the SSL certificate for HTTPS requests
+
+            *cert* client side certificates for HTTPS requests
+
+        :param args: positional arguments to pass to :class:`consul.base.Consul`
+        :param kwargs: keyword arguments, with *cert*, *ca_cert* and *token* removed, passed to
+                       :class:`consul.base.Consul`
+        """
         self._cert = kwargs.pop('cert', None)
         self._ca_cert = kwargs.pop('ca_cert', None)
         self.token = kwargs.get('token')
@@ -202,8 +233,8 @@ def service_name_from_scope_name(scope_name: str) -> str:
 
 class Consul(AbstractDCS):
 
-    def __init__(self, config: Dict[str, Any]) -> None:
+    def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None:
-        super(Consul, self).__init__(config)
+        super(Consul, self).__init__(config, mpp)
         self._base_path = self._base_path[1:]
         self._scope = config['scope']
         self._session = None
@@ -353,23 +384,8 @@ class Consul(AbstractDCS):
         history = history and TimelineHistory.from_node(history['ModifyIndex'], history['Value'])
 
         # get last known leader lsn and slots
-        status = nodes.get(self._STATUS)
+        status = nodes.get(self._STATUS) or nodes.get(self._LEADER_OPTIME)
-        if status:
+        status = Status.from_node(status and status['Value'])
-            try:
-                status = json.loads(status['Value'])
-                last_lsn = status.get(self._OPTIME)
-                slots = status.get('slots')
-            except Exception:
-                slots = last_lsn = None
-        else:
-            last_lsn = nodes.get(self._LEADER_OPTIME)
-            last_lsn = last_lsn and last_lsn['Value']
-            slots = None
-
-        try:
-            last_lsn = int(last_lsn or '')
-        except Exception:
-            last_lsn = 0
 
         # get list of members
         members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
@@ -398,29 +414,41 @@ class Consul(AbstractDCS):
         except Exception:
             failsafe = None
 
-        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
+        return Cluster(initialize, config, leader, status, members, failover, sync, history, failsafe)
 
     @property
     def _consistency(self) -> str:
         return 'consistent' if self._ctl else self._client.consistency
 
-    def _cluster_loader(self, path: str) -> Cluster:
+    def _postgresql_cluster_loader(self, path: str) -> Cluster:
+        """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster.
+
+        :param path: the path in DCS where to load :class:`Cluster` from.
+
+        :returns: :class:`Cluster` instance.
+        """
         _, results = self.retry(self._client.kv.get, path, recurse=True, consistency=self._consistency)
         if results is None:
-            raise NotFound
+            return Cluster.empty()
-        nodes = {}
+        nodes: Dict[str, Dict[str, Any]] = {}
         for node in results:
             node['Value'] = (node['Value'] or b'').decode('utf-8')
             nodes[node['Key'][len(path):]] = node
 
         return self._cluster_from_nodes(nodes)
 
-    def _citus_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+    def _mpp_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+        """Load and build all PostgreSQL clusters from a single MPP cluster.
+
+        :param path: the path in DCS where to load Cluster(s) from.
+
+        :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values.
+        """
         _, results = self.retry(self._client.kv.get, path, recurse=True, consistency=self._consistency)
         clusters: Dict[int, Dict[str, Cluster]] = defaultdict(dict)
         for node in results or []:
             key = node['Key'][len(path):].split('/', 1)
-            if len(key) == 2 and citus_group_re.match(key[0]):
+            if len(key) == 2 and self._mpp.group_re.match(key[0]):
                 node['Value'] = (node['Value'] or b'').decode('utf-8')
                 clusters[int(key[0])][key[1]] = node
         return {group: self._cluster_from_nodes(nodes) for group, nodes in clusters.items()}
@@ -430,8 +458,6 @@ class Consul(AbstractDCS):
     ) -> Union[Cluster, Dict[int, Cluster]]:
         try:
             return loader(path)
-        except NotFound:
-            return Cluster.empty()
         except Exception:
             logger.exception('get_cluster')
             raise ConsulError('Consul is not responding properly')
@@ -551,14 +577,17 @@ class Consul(AbstractDCS):
         try:
             return retry(self._client.kv.put, self.leader_path, self._name, acquire=self._session)
         except InvalidSession:
-            logger.error('Our session disappeared from Consul. Will try to get a new one and retry attempt')
             self._session = None
-            retry.ensure_deadline(0)
+
+            if not retry.ensure_deadline(0):
+                logger.error('Our session disappeared from Consul. Deadline exceeded, giving up')
+                return False
+
+            logger.error('Our session disappeared from Consul. Will try to get a new one and retry attempt')
 
             retry(self._do_refresh_session)
 
             retry.ensure_deadline(1, ConsulError('_do_attempt_to_acquire_leader timeout'))
-
             return retry(self._client.kv.put, self.leader_path, self._name, acquire=self._session)
 
     @catch_return_false_exception
@@ -643,12 +672,8 @@ class Consul(AbstractDCS):
         return self._client.kv.put(self.history_path, value)
 
     @catch_consul_errors
-    def _delete_leader(self) -> bool:
+    def _delete_leader(self, leader: Leader) -> bool:
-        cluster = self.cluster
+        return self._client.kv.delete(self.leader_path, cas=int(leader.version))
-        if cluster and isinstance(cluster.leader, Leader) and\
-                cluster.leader.name == self._name and isinstance(cluster.leader.version, int):
-            return self._client.kv.delete(self.leader_path, cas=cluster.leader.version)
-        return True
 
     @catch_consul_errors
     def set_sync_state_value(self, value: str, version: Optional[int] = None) -> Union[int, bool]:
@@ -657,7 +682,7 @@ class Consul(AbstractDCS):
         if ret:  # We have no other choise, only read after write :(
             if not retry.ensure_deadline(0.5):
                 return False
-            _, ret = self.retry(self._client.kv.get, self.sync_path)
+            _, ret = self.retry(self._client.kv.get, self.sync_path, consistency='consistent')
             if ret and (ret.get('Value') or b'').decode('utf-8') == value:
                 return ret['ModifyIndex']
         return False

patroni/dcs/etcd.py

@@ -21,9 +21,10 @@ from urllib.parse import urlparse
 from urllib3 import Timeout
 from urllib3.exceptions import HTTPError, ReadTimeoutError, ProtocolError
 
-from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState,\
+from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, Status, SyncState, \
-    TimelineHistory, ReturnFalseException, catch_return_false_exception, citus_group_re
+    TimelineHistory, ReturnFalseException, catch_return_false_exception
 from ..exceptions import DCSError
+from ..postgresql.mpp import AbstractMPP
 from ..request import get as requests_get
 from ..utils import Retry, RetryFailedError, split_host_port, uri, USER_AGENT
 if TYPE_CHECKING:  # pragma: no cover
@@ -470,9 +471,9 @@ class EtcdClient(AbstractEtcdClientWithFailover):
 
 class AbstractEtcd(AbstractDCS):
 
-    def __init__(self, config: Dict[str, Any], client_cls: Type[AbstractEtcdClientWithFailover],
+    def __init__(self, config: Dict[str, Any], mpp: AbstractMPP, client_cls: Type[AbstractEtcdClientWithFailover],
                  retry_errors_cls: Union[Type[Exception], Tuple[Type[Exception], ...]]) -> None:
-        super(AbstractEtcd, self).__init__(config)
+        super(AbstractEtcd, self).__init__(config, mpp)
         self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
                             retry_exceptions=retry_errors_cls)
         self._ttl = int(config.get('ttl') or 30)
@@ -645,8 +646,8 @@ def catch_etcd_errors(func: Callable[..., Any]) -> Any:
 
 class Etcd(AbstractEtcd):
 
-    def __init__(self, config: Dict[str, Any]) -> None:
+    def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None:
-        super(Etcd, self).__init__(config, EtcdClient, (etcd.EtcdLeaderElectionInProgress, EtcdRaftInternal))
+        super(Etcd, self).__init__(config, mpp, EtcdClient, (etcd.EtcdLeaderElectionInProgress, EtcdRaftInternal))
         self.__do_not_watch = False
 
     @property
@@ -677,23 +678,8 @@ class Etcd(AbstractEtcd):
         history = history and TimelineHistory.from_node(history.modifiedIndex, history.value)
 
         # get last know leader lsn and slots
-        status = nodes.get(self._STATUS)
+        status = nodes.get(self._STATUS) or nodes.get(self._LEADER_OPTIME)
-        if status:
+        status = Status.from_node(status and status.value)
-            try:
-                status = json.loads(status.value)
-                last_lsn = status.get(self._OPTIME)
-                slots = status.get('slots')
-            except Exception:
-                slots = last_lsn = None
-        else:
-            last_lsn = nodes.get(self._LEADER_OPTIME)
-            last_lsn = last_lsn and last_lsn.value
-            slots = None
-
-        try:
-            last_lsn = int(last_lsn or '')
-        except Exception:
-            last_lsn = 0
 
         # get list of members
         members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
@@ -722,19 +708,38 @@ class Etcd(AbstractEtcd):
         except Exception:
             failsafe = None
 
-        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
+        return Cluster(initialize, config, leader, status, members, failover, sync, history, failsafe)
 
-    def _cluster_loader(self, path: str) -> Cluster:
+    def _postgresql_cluster_loader(self, path: str) -> Cluster:
-        result = self.retry(self._client.read, path, recursive=True, quorum=self._ctl)
+        """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster.
+
+        :param path: the path in DCS where to load :class:`Cluster` from.
+
+        :returns: :class:`Cluster` instance.
+        """
+        try:
+            result = self.retry(self._client.read, path, recursive=True, quorum=self._ctl)
+        except etcd.EtcdKeyNotFound:
+            return Cluster.empty()
         nodes = {node.key[len(result.key):].lstrip('/'): node for node in result.leaves}
         return self._cluster_from_nodes(result.etcd_index, nodes)
 
-    def _citus_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+    def _mpp_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+        """Load and build all PostgreSQL clusters from a single MPP cluster.
+
+        :param path: the path in DCS where to load Cluster(s) from.
+
+        :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values.
+        """
+        try:
+            result = self.retry(self._client.read, path, recursive=True, quorum=self._ctl)
+        except etcd.EtcdKeyNotFound:
+            return {}
+
         clusters: Dict[int, Dict[str, etcd.EtcdResult]] = defaultdict(dict)
-        result = self.retry(self._client.read, path, recursive=True, quorum=self._ctl)
         for node in result.leaves:
             key = node.key[len(result.key):].lstrip('/').split('/', 1)
-            if len(key) == 2 and citus_group_re.match(key[0]):
+            if len(key) == 2 and self._mpp.group_re.match(key[0]):
                 clusters[int(key[0])][key[1]] = node
         return {group: self._cluster_from_nodes(result.etcd_index, nodes) for group, nodes in clusters.items()}
 
@@ -744,8 +749,6 @@ class Etcd(AbstractEtcd):
         cluster = None
         try:
             cluster = loader(path)
-        except etcd.EtcdKeyNotFound:
-            cluster = Cluster.empty()
         except Exception as e:
             self._handle_exception(e, 'get_cluster', raise_ex=EtcdError('Etcd is not responding properly'))
         self._has_failed = False
@@ -809,7 +812,7 @@ class Etcd(AbstractEtcd):
         return bool(self.retry(self._client.write, self.initialize_path, sysid, prevExist=(not create_new)))
 
     @catch_etcd_errors
-    def _delete_leader(self) -> bool:
+    def _delete_leader(self, leader: Leader) -> bool:
         return bool(self._client.delete(self.leader_path, prevValue=self._name))
 
     @catch_etcd_errors

patroni/dcs/etcd3.py

@@ -15,10 +15,11 @@ from urllib3.exceptions import ReadTimeoutError, ProtocolError
 from threading import Condition, Lock, Thread
 from typing import Any, Callable, Collection, Dict, Iterator, List, Optional, Tuple, Type, TYPE_CHECKING, Union
 
-from . import ClusterConfig, Cluster, Failover, Leader, Member, SyncState,\
+from . import ClusterConfig, Cluster, Failover, Leader, Member, Status, SyncState, \
-    TimelineHistory, catch_return_false_exception, citus_group_re
+    TimelineHistory, catch_return_false_exception
 from .etcd import AbstractEtcdClientWithFailover, AbstractEtcd, catch_etcd_errors, DnsCachingResolver, Retry
 from ..exceptions import DCSError, PatroniException
+from ..postgresql.mpp import AbstractMPP
 from ..utils import deep_compare, enable_keepalive, iter_response_objects, RetryFailedError, USER_AGENT
 
 logger = logging.getLogger(__name__)
@@ -124,6 +125,10 @@ class AuthFailed(InvalidArgument):
     error = "etcdserver: authentication failed, invalid user ID or password"
 
 
+class AuthOldRevision(InvalidArgument):
+    error = "etcdserver: revision of auth store is old"
+
+
 class PermissionDenied(Etcd3ClientError):
     code = GRPCCode.PermissionDenied
     error = "etcdserver: permission denied"
@@ -204,8 +209,9 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
     ERROR_CLS = Etcd3Error
 
     def __init__(self, config: Dict[str, Any], dns_resolver: DnsCachingResolver, cache_ttl: int = 300) -> None:
+        self._reauthenticate = False
         self._token = None
-        self._cluster_version: Tuple[int] = tuple()
+        self._cluster_version: Tuple[int, ...] = tuple()
         super(Etcd3Client, self).__init__({**config, 'version_prefix': '/v3beta'}, dns_resolver, cache_ttl)
 
         try:
@@ -228,7 +234,7 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
         return self.http.urlopen
 
     def _handle_server_response(self, response: urllib3.response.HTTPResponse) -> Dict[str, Any]:
-        data: Union[bytes, str] = response.data
+        data = response.data
         try:
             data = data.decode('utf-8')
             ret: Dict[str, Any] = json.loads(data)
@@ -282,7 +288,7 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
         fields['retry'] = retry
         return self.api_execute(self.version_prefix + method, self._MPOST, fields)
 
-    def authenticate(self) -> bool:
+    def authenticate(self, *, retry: Optional[Retry] = None) -> bool:
         if self._use_proxies and not self._cluster_version:
             kwargs = self._prepare_common_parameters(1)
             self._ensure_version_prefix(self._base_uri, **kwargs)
@@ -291,7 +297,7 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
         logger.info('Trying to authenticate on Etcd...')
         old_token, self._token = self._token, None
         try:
-            response = self.call_rpc('/auth/authenticate', {'name': self.username, 'password': self.password})
+            response = self.call_rpc('/auth/authenticate', {'name': self.username, 'password': self.password}, retry)
         except AuthNotEnabled:
             logger.info('Etcd authentication is not enabled')
             self._token = None
@@ -302,48 +308,64 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
             self._token = response.get('token')
         return old_token != self._token
 
-    def handle_auth_errors(self: 'Etcd3Client', func: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
+    def handle_auth_errors(self: 'Etcd3Client', func: Callable[..., Any], *args: Any,
-        def retry(ex: Exception) -> Any:
+                           retry: Optional[Retry] = None, **kwargs: Any) -> Any:
-            if self.username and self.password:
+        reauthenticated = False
-                self.authenticate()
+        exc = None
-                return func(self, *args, **kwargs)
+        while True:
-            else:
+            if self._reauthenticate:
-                logger.fatal('Username or password not set, authentication is not possible')
+                if self.username and self.password:
-                raise ex
+                    self.authenticate(retry=retry)
+                    self._reauthenticate = False
+                else:
+                    msg = 'Username or password not set, authentication is not possible'
+                    logger.fatal(msg)
+                    raise exc or Etcd3Exception(msg)
+                reauthenticated = True
 
-        try:
+            try:
-            return func(self, *args, **kwargs)
+                return func(self, *args, retry=retry, **kwargs)
-        except (UserEmpty, PermissionDenied) as e:  # no token provided
+            except (UserEmpty, PermissionDenied) as e:  # no token provided
-            # PermissionDenied is raised on 3.0 and 3.1
+                # PermissionDenied is raised on 3.0 and 3.1
-            if self._cluster_version < (3, 3) and (not isinstance(e, PermissionDenied)
+                if self._cluster_version < (3, 3) and (not isinstance(e, PermissionDenied)
-                                                   or self._cluster_version < (3, 2)):
+                                                       or self._cluster_version < (3, 2)):
-                raise UnsupportedEtcdVersion('Authentication is required by Etcd cluster but not '
+                    raise UnsupportedEtcdVersion('Authentication is required by Etcd cluster but not '
-                                             'supported on version lower than 3.3.0. Cluster version: '
+                                                 'supported on version lower than 3.3.0. Cluster version: '
-                                             '{0}'.format('.'.join(map(str, self._cluster_version))))
+                                                 '{0}'.format('.'.join(map(str, self._cluster_version))))
-            return retry(e)
+                exc = e
-        except InvalidAuthToken as e:
+            except InvalidAuthToken as e:
-            logger.error('Invalid auth token: %s', self._token)
+                logger.error('Invalid auth token: %s', self._token)
-            return retry(e)
+                exc = e
+            except AuthOldRevision as e:
+                logger.error('Auth token is for old revision of auth store')
+                exc = e
+            self._reauthenticate = True
+            if retry:
+                logger.error('retry = %s', retry)
+                retry.ensure_deadline(0.5, exc)
+            elif reauthenticated:
+                raise exc
 
     @_handle_auth_errors
     def range(self, key: str, range_end: Union[bytes, str, None] = None, serializable: bool = True,
-              retry: Optional[Retry] = None) -> Dict[str, Any]:
+              *, retry: Optional[Retry] = None) -> Dict[str, Any]:
         params = build_range_request(key, range_end)
         params['serializable'] = serializable  # For better performance. We can tolerate stale reads
         return self.call_rpc('/kv/range', params, retry)
 
-    def prefix(self, key: str, serializable: bool = True, retry: Optional[Retry] = None) -> Dict[str, Any]:
+    def prefix(self, key: str, serializable: bool = True, *, retry: Optional[Retry] = None) -> Dict[str, Any]:
-        return self.range(key, prefix_range_end(key), serializable, retry)
+        return self.range(key, prefix_range_end(key), serializable, retry=retry)
 
     @_handle_auth_errors
-    def lease_grant(self, ttl: int, retry: Optional[Retry] = None) -> str:
+    def lease_grant(self, ttl: int, *, retry: Optional[Retry] = None) -> str:
         return self.call_rpc('/lease/grant', {'TTL': ttl}, retry)['ID']
 
-    def lease_keepalive(self, ID: str, retry: Optional[Retry] = None) -> Optional[str]:
+    def lease_keepalive(self, ID: str, *, retry: Optional[Retry] = None) -> Optional[str]:
         return self.call_rpc('/lease/keepalive', {'ID': ID}, retry).get('result', {}).get('TTL')
 
+    @_handle_auth_errors
     def txn(self, compare: Dict[str, Any], success: Dict[str, Any],
-            failure: Optional[Dict[str, Any]] = None, retry: Optional[Retry] = None) -> Dict[str, Any]:
+            failure: Optional[Dict[str, Any]] = None, *, retry: Optional[Retry] = None) -> Dict[str, Any]:
         fields = {'compare': [compare], 'success': [success]}
         if failure:
             fields['failure'] = [failure]
@@ -352,7 +374,7 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
 
     @_handle_auth_errors
     def put(self, key: str, value: str, lease: Optional[str] = None, create_revision: Optional[str] = None,
-            mod_revision: Optional[str] = None, retry: Optional[Retry] = None) -> Dict[str, Any]:
+            mod_revision: Optional[str] = None, *, retry: Optional[Retry] = None) -> Dict[str, Any]:
         fields = {'key': base64_encode(key), 'value': base64_encode(value)}
         if lease:
             fields['lease'] = lease
@@ -367,14 +389,14 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
 
     @_handle_auth_errors
     def deleterange(self, key: str, range_end: Union[bytes, str, None] = None,
-                    mod_revision: Optional[str] = None, retry: Optional[Retry] = None) -> Dict[str, Any]:
+                    mod_revision: Optional[str] = None, *, retry: Optional[Retry] = None) -> Dict[str, Any]:
         fields = build_range_request(key, range_end)
         if mod_revision is None:
             return self.call_rpc('/kv/deleterange', fields, retry)
         compare = {'target': 'MOD', 'mod_revision': mod_revision, 'key': fields['key']}
         return self.txn(compare, {'request_delete_range': fields}, retry=retry)
 
-    def deleteprefix(self, key: str, retry: Optional[Retry] = None) -> Dict[str, Any]:
+    def deleteprefix(self, key: str, *, retry: Optional[Retry] = None) -> Dict[str, Any]:
         return self.deleterange(key, prefix_range_end(key), retry=retry)
 
     def watchrange(self, key: str, range_end: Union[bytes, str, None] = None,
@@ -574,12 +596,6 @@ class PatroniEtcd3Client(Etcd3Client):
         super(PatroniEtcd3Client, self).set_base_uri(value)
         self._restart_watcher()
 
-    def authenticate(self) -> bool:
-        ret = super(PatroniEtcd3Client, self).authenticate()
-        if ret:
-            self._restart_watcher()
-        return ret
-
     def _wait_cache(self, timeout: float) -> None:
         stop_time = time.time() + timeout
         while self._kv_cache and not self._kv_cache.is_ready():
@@ -630,11 +646,22 @@ class PatroniEtcd3Client(Etcd3Client):
 
         return ret
 
+    def txn(self, compare: Dict[str, Any], success: Dict[str, Any],
+            failure: Optional[Dict[str, Any]] = None, *, retry: Optional[Retry] = None) -> Dict[str, Any]:
+        ret = super(PatroniEtcd3Client, self).txn(compare, success, failure, retry=retry)
+        # Here we abuse the fact that the `failure` is only set in the call from update_leader().
+        # In all other cases the txn() call failure may be an indicator of a stale cache,
+        # and therefore we want to restart watcher.
+        if not failure and not ret:
+            self._restart_watcher()
+        return ret
+
 
 class Etcd3(AbstractEtcd):
 
-    def __init__(self, config: Dict[str, Any]) -> None:
+    def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None:
-        super(Etcd3, self).__init__(config, PatroniEtcd3Client, (DeadlineExceeded, Unavailable, FailedPrecondition))
+        super(Etcd3, self).__init__(config, mpp, PatroniEtcd3Client,
+                                    (DeadlineExceeded, Unavailable, FailedPrecondition))
         self.__do_not_watch = False
         self._lease = None
         self._last_lease_refresh = 0
@@ -666,12 +693,12 @@ class Etcd3(AbstractEtcd):
         if not force and self._lease and self._last_lease_refresh + self._loop_wait > time.time():
             return False
 
-        if self._lease and not self._client.lease_keepalive(self._lease, retry):
+        if self._lease and not self._client.lease_keepalive(self._lease, retry=retry):
             self._lease = None
 
         ret = not self._lease
         if ret:
-            self._lease = self._client.lease_grant(self._ttl, retry)
+            self._lease = self._client.lease_grant(self._ttl, retry=retry)
 
         self._last_lease_refresh = time.time()
         return ret
@@ -693,7 +720,11 @@ class Etcd3(AbstractEtcd):
 
     @property
     def cluster_prefix(self) -> str:
-        return self._base_path + '/' if self.is_citus_coordinator() else self.client_path('')
+        """Construct the cluster prefix for the cluster.
+
+        :returns: path in the DCS under which we store information about this Patroni cluster.
+        """
+        return self._base_path + '/' if self.is_mpp_coordinator() else self.client_path('')
 
     @staticmethod
     def member(node: Dict[str, str]) -> Member:
@@ -713,23 +744,8 @@ class Etcd3(AbstractEtcd):
         history = history and TimelineHistory.from_node(history['mod_revision'], history['value'])
 
         # get last know leader lsn and slots
-        status = nodes.get(self._STATUS)
+        status = nodes.get(self._STATUS) or nodes.get(self._LEADER_OPTIME)
-        if status:
+        status = Status.from_node(status and status['value'])
-            try:
-                status = json.loads(status['value'])
-                last_lsn = status.get(self._OPTIME)
-                slots = status.get('slots')
-            except Exception:
-                slots = last_lsn = None
-        else:
-            last_lsn = nodes.get(self._LEADER_OPTIME)
-            last_lsn = last_lsn and last_lsn['value']
-            slots = None
-
-        try:
-            last_lsn = int(last_lsn or '')
-        except Exception:
-            last_lsn = 0
 
         # get list of members
         members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
@@ -760,20 +776,32 @@ class Etcd3(AbstractEtcd):
         except Exception:
             failsafe = None
 
-        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
+        return Cluster(initialize, config, leader, status, members, failover, sync, history, failsafe)
 
-    def _cluster_loader(self, path: str) -> Cluster:
+    def _postgresql_cluster_loader(self, path: str) -> Cluster:
+        """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster.
+
+        :param path: the path in DCS where to load :class:`Cluster` from.
+
+        :returns: :class:`Cluster` instance.
+        """
         nodes = {node['key'][len(path):]: node
                  for node in self._client.get_cluster(path)
                  if node['key'].startswith(path)}
         return self._cluster_from_nodes(nodes)
 
-    def _citus_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+    def _mpp_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+        """Load and build all PostgreSQL clusters from a single MPP cluster.
+
+        :param path: the path in DCS where to load Cluster(s) from.
+
+        :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values.
+        """
         clusters: Dict[int, Dict[str, Dict[str, Any]]] = defaultdict(dict)
         path = self._base_path + '/'
         for node in self._client.get_cluster(path):
             key = node['key'][len(path):].split('/', 1)
-            if len(key) == 2 and citus_group_re.match(key[0]):
+            if len(key) == 2 and self._mpp.group_re.match(key[0]):
                 clusters[int(key[0])][key[1]] = node
         return {group: self._cluster_from_nodes(nodes) for group, nodes in clusters.items()}
 
@@ -825,14 +853,16 @@ class Etcd3(AbstractEtcd):
         try:
             return _retry(self._client.put, self.leader_path, self._name, self._lease, create_revision='0')
         except LeaseNotFound:
-            logger.error('Our lease disappeared from Etcd. Will try to get a new one and retry attempt')
             self._lease = None
-            retry.ensure_deadline(0)
+            if not retry.ensure_deadline(0):
+                logger.error('Our lease disappeared from Etcd. Deadline exceeded, giving up')
+                return False
+
+            logger.error('Our lease disappeared from Etcd. Will try to get a new one and retry attempt')
 
             _retry(self._do_refresh_lease)
 
             retry.ensure_deadline(1, Etcd3Error('_do_attempt_to_acquire_leader timeout'))
-
             return _retry(self._client.put, self.leader_path, self._name, self._lease, create_revision='0')
 
     @catch_return_false_exception
@@ -902,11 +932,10 @@ class Etcd3(AbstractEtcd):
         return self.retry(self._client.put, self.initialize_path, sysid, create_revision='0' if create_new else None)
 
     @catch_etcd_errors
-    def _delete_leader(self) -> bool:
+    def _delete_leader(self, leader: Leader) -> bool:
-        cluster = self.cluster
+        fields = build_range_request(self.leader_path)
-        if cluster and isinstance(cluster.leader, Leader) and cluster.leader.name == self._name:
+        compare = {'key': fields['key'], 'target': 'VALUE', 'value': base64_encode(self._name)}
-            return self._client.deleterange(self.leader_path, mod_revision=cluster.leader.version)
+        return bool(self._client.txn(compare, {'request_delete_range': fields}))
-        return True
 
     @catch_etcd_errors
     def cancel_initialization(self) -> bool:

patroni/dcs/exhibitor.py

@@ -7,6 +7,7 @@ from typing import Any, Callable, Dict, List, Union
 
 from . import Cluster
 from .zookeeper import ZooKeeper
+from ..postgresql.mpp import AbstractMPP
 from ..request import get as requests_get
 from ..utils import uri
 
@@ -66,10 +67,10 @@ class ExhibitorEnsembleProvider(object):
 
 class Exhibitor(ZooKeeper):
 
-    def __init__(self, config: Dict[str, Any]) -> None:
+    def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None:
         interval = config.get('poll_interval', 300)
         self._ensemble_provider = ExhibitorEnsembleProvider(config['hosts'], config['port'], poll_interval=interval)
-        super(Exhibitor, self).__init__({**config, 'hosts': self._ensemble_provider.zookeeper_hosts})
+        super(Exhibitor, self).__init__({**config, 'hosts': self._ensemble_provider.zookeeper_hosts}, mpp)
 
     def _load_cluster(
             self, path: str, loader: Callable[[str], Union[Cluster, Dict[int, Cluster]]]

patroni/dcs/kubernetes.py

@@ -19,10 +19,10 @@ from urllib3.exceptions import HTTPError
 from threading import Condition, Lock, Thread
 from typing import Any, Callable, Collection, Dict, List, Optional, Tuple, Type, Union, TYPE_CHECKING
 
-from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState,\
+from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, Status, SyncState, TimelineHistory
-    TimelineHistory, CITUS_COORDINATOR_GROUP_ID, citus_group_re
 from ..exceptions import DCSError
-from ..utils import deep_compare, iter_response_objects, keepalive_socket_options,\
+from ..postgresql.mpp import AbstractMPP
+from ..utils import deep_compare, iter_response_objects, keepalive_socket_options, \
     Retry, RetryFailedError, tzutc, uri, USER_AGENT
 if TYPE_CHECKING:  # pragma: no cover
     from ..config import Config
@@ -134,6 +134,8 @@ class K8sConfig(object):
             config: Dict[str, Any] = yaml.safe_load(f)
 
         context = context or config['current-context']
+        if TYPE_CHECKING:  # pragma: no cover
+            assert isinstance(context, str)
         context_value = self._get_by_name(config, 'context', context)
         if TYPE_CHECKING:  # pragma: no cover
             assert isinstance(context_value, dict)
@@ -744,18 +746,20 @@ class ObjectCache(Thread):
 
 class Kubernetes(AbstractDCS):
 
-    _CITUS_LABEL = 'citus-group'
+    def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None:
-
-    def __init__(self, config: Dict[str, Any]) -> None:
         self._labels = deepcopy(config['labels'])
         self._labels[config.get('scope_label', 'cluster-name')] = config['scope']
         self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
         self._namespace = config.get('namespace') or 'default'
         self._role_label = config.get('role_label', 'role')
+        self._leader_label_value = config.get('leader_label_value', 'master')
+        self._follower_label_value = config.get('follower_label_value', 'replica')
+        self._standby_leader_label_value = config.get('standby_leader_label_value', 'master')
+        self._tmp_role_label = config.get('tmp_role_label')
         self._ca_certs = os.environ.get('PATRONI_KUBERNETES_CACERT', config.get('cacert')) or SERVICE_CERT_FILENAME
-        super(Kubernetes, self).__init__({**config, 'namespace': ''})
+        super(Kubernetes, self).__init__({**config, 'namespace': ''}, mpp)
-        if self._citus_group:
+        if self._mpp.is_enabled():
-            self._labels[self._CITUS_LABEL] = self._citus_group
+            self._labels[self._mpp.k8s_group_label] = str(self._mpp.group)
 
         self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
                             retry_exceptions=KubernetesRetriableException)
@@ -765,8 +769,7 @@ class Kubernetes(AbstractDCS):
         except k8s_config.ConfigException:
             k8s_config.load_kube_config(context=config.get('context', 'kind-kind'))
 
-        pod_ip = config.get('pod_ip')
+        self.__ips: List[str] = [] if self._ctl else [config.get('pod_ip', '')]
-        self.__ips: List[str] = [] if self._ctl or not isinstance(pod_ip, str) else [pod_ip]
         self.__ports: List[K8sObject] = []
         ports: List[Dict[str, Any]] = config.get('ports', [{}])
         for p in ports:
@@ -830,7 +833,7 @@ class Kubernetes(AbstractDCS):
         self._api.configure_timeouts(self.loop_wait, self._retry.deadline, self.ttl)
 
         # retriable_http_codes supposed to be either int, list of integers or comma-separated string with integers.
-        retriable_http_codes = config.get('retriable_http_codes', [])
+        retriable_http_codes: Union[str, List[Union[str, int]]] = config.get('retriable_http_codes', [])
         if not isinstance(retriable_http_codes, list):
             retriable_http_codes = [c.strip() for c in str(retriable_http_codes).split(',')]
 
@@ -882,18 +885,8 @@ class Kubernetes(AbstractDCS):
             self._leader_resource_version = metadata.resource_version if metadata else None
         annotations: Dict[str, str] = metadata and metadata.annotations or {}
 
-        # get last known leader lsn
+        # get last known leader lsn and slots
-        try:
+        status = Status.from_node(annotations)
-            last_lsn = int(annotations.get(self._OPTIME, ''))
-        except Exception:
-            last_lsn = 0
-
-        # get permanent slots state (confirmed_flush_lsn)
-        slots = annotations.get('slots')
-        try:
-            slots = json.loads(annotations.get('slots', ''))
-        except Exception:
-            slots = None
 
         # get failsafe topology
         try:
@@ -939,22 +932,34 @@ class Kubernetes(AbstractDCS):
         metadata = sync and sync.metadata
         sync = SyncState.from_node(metadata and metadata.resource_version, metadata and metadata.annotations)
 
-        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
+        return Cluster(initialize, config, leader, status, members, failover, sync, history, failsafe)
 
-    def _cluster_loader(self, path: Dict[str, Any]) -> Cluster:
+    def _postgresql_cluster_loader(self, path: Dict[str, Any]) -> Cluster:
+        """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster.
+
+        :param path: the path in DCS where to load :class:`Cluster` from.
+
+        :returns: :class:`Cluster` instance.
+        """
         return self._cluster_from_nodes(path['group'], path['nodes'], path['pods'].values())
 
-    def _citus_cluster_loader(self, path: Dict[str, Any]) -> Dict[int, Cluster]:
+    def _mpp_cluster_loader(self, path: Dict[str, Any]) -> Dict[int, Cluster]:
+        """Load and build all PostgreSQL clusters from a single MPP cluster.
+
+        :param path: the path in DCS where to load Cluster(s) from.
+
+        :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values.
+        """
         clusters: Dict[str, Dict[str, Dict[str, K8sObject]]] = defaultdict(lambda: defaultdict(dict))
 
         for name, pod in path['pods'].items():
-            group = pod.metadata.labels.get(self._CITUS_LABEL)
+            group = pod.metadata.labels.get(self._mpp.k8s_group_label)
-            if group and citus_group_re.match(group):
+            if group and self._mpp.group_re.match(group):
                 clusters[group]['pods'][name] = pod
 
         for name, kind in path['nodes'].items():
-            group = kind.metadata.labels.get(self._CITUS_LABEL)
+            group = kind.metadata.labels.get(self._mpp.k8s_group_label)
-            if group and citus_group_re.match(group):
+            if group and self._mpp.group_re.match(group):
                 clusters[group]['nodes'][name] = kind
         return {int(group): self._cluster_from_nodes(group, value['nodes'], value['pods'].values())
                 for group, value in clusters.items()}
@@ -970,9 +975,9 @@ class Kubernetes(AbstractDCS):
             with self._condition:
                 self._wait_caches(stop_time)
                 pods = {name: pod for name, pod in self._pods.copy().items()
-                        if not group or pod.metadata.labels.get(self._CITUS_LABEL) == group}
+                        if not group or pod.metadata.labels.get(self._mpp.k8s_group_label) == group}
                 nodes = {name: kind for name, kind in self._kinds.copy().items()
-                         if not group or kind.metadata.labels.get(self._CITUS_LABEL) == group}
+                         if not group or kind.metadata.labels.get(self._mpp.k8s_group_label) == group}
             return loader({'group': group, 'pods': pods, 'nodes': nodes})
         except Exception:
             logger.exception('get_cluster')
@@ -981,17 +986,24 @@ class Kubernetes(AbstractDCS):
     def _load_cluster(
             self, path: str, loader: Callable[[Any], Union[Cluster, Dict[int, Cluster]]]
     ) -> Union[Cluster, Dict[int, Cluster]]:
-        group = self._citus_group if path == self.client_path('') else None
+        group = str(self._mpp.group) if self._mpp.is_enabled() and path == self.client_path('') else None
         return self.__load_cluster(group, loader)
 
-    def get_citus_coordinator(self) -> Optional[Cluster]:
+    def get_mpp_coordinator(self) -> Optional[Cluster]:
+        """Load the PostgreSQL cluster for the MPP Coordinator.
+
+        .. note::
+            This method is only executed on the worker nodes to find the coordinator.
+
+        :returns: Select :class:`Cluster` instance associated with the MPP Coordinator group ID.
+        """
         try:
-            ret = self.__load_cluster(str(CITUS_COORDINATOR_GROUP_ID), self._cluster_loader)
+            ret = self.__load_cluster(str(self._mpp.coordinator_group_id), self._postgresql_cluster_loader)
             if TYPE_CHECKING:  # pragma: no cover
                 assert isinstance(ret, Cluster)
             return ret
         except Exception as e:
-            logger.error('Failed to load Citus coordinator cluster from Kubernetes: %r', e)
+            logger.error('Failed to load %s coordinator cluster from Kubernetes: %r', self._mpp.type, e)
 
     @staticmethod
     def compare_ports(p1: K8sObject, p2: K8sObject) -> bool:
@@ -1063,6 +1075,27 @@ class Kubernetes(AbstractDCS):
     def _patch_or_create(self, name: str, annotations: Dict[str, Any],
                          resource_version: Optional[str] = None, patch: bool = False,
                          retry: Optional[Callable[..., Any]] = None, ips: Optional[List[str]] = None) -> K8sObject:
+        """Patch or create K8s object, Endpoint or ConfigMap.
+
+        :param name: the name of the object.
+        :param annotations: mapping of annotations that we want to create/update.
+        :param resource_version: object should be updated only if the ``resource_version`` matches provided value.
+        :param patch: ``True`` if we know in advance that the object already exists and we should patch it.
+        :param retry: a callable that will take care of retries
+        :param ips: IP address that we want to put to the subsets of the endpoint. Could have following values:
+
+                    * ``None`` - when we don't need to touch subset;
+                    * ``[]`` - to set subsets to the empty list, when :meth:`delete_leader` method is called;
+
+                    * ``['ip.add.re.ss']`` - when we want to make sure that the subsets of the leader endpoint
+                      contains the IP address of the leader, that we get from the ``kubernetes.pod_ip``;
+
+                    * ``['']`` - when we want to make sure that the subsets of the leader endpoint contains the IP
+                      address of the leader, but ``kubernetes.pod_ip`` configuration is missing. In this case we will
+                      try to take the IP address of the Pod which name matches ``name`` from the config file.
+
+        :returns: the new :class:`V1Endpoints` or :class:`V1ConfigMap` object, that was created or updated.
+        """
         metadata = {'namespace': self._namespace, 'name': name, 'labels': self._labels, 'annotations': annotations}
         if patch or resource_version:
             if resource_version is not None:
@@ -1075,9 +1108,10 @@ class Kubernetes(AbstractDCS):
             metadata['annotations'] = {k: v for k, v in annotations.items() if v is not None}
 
         metadata = k8s_client.V1ObjectMeta(**metadata)
-        if ips is not None and self._api.use_endpoints:
+        if self._api.use_endpoints:
             endpoints = {'metadata': metadata}
-            self._map_subsets(endpoints, ips)
+            if ips is not None:
+                self._map_subsets(endpoints, ips)
             body = k8s_client.V1Endpoints(**endpoints)
         else:
             body = k8s_client.V1ConfigMap(metadata=metadata)
@@ -1134,6 +1168,13 @@ class Kubernetes(AbstractDCS):
         """Unused"""
         raise NotImplementedError  # pragma: no cover
 
+    def write_leader_optime(self, last_lsn: int) -> None:
+        """Write value for WAL LSN to ``optime`` annotation of the leader object.
+
+        :param last_lsn: absolute WAL LSN in bytes.
+        """
+        self.patch_or_create(self.leader_path, {self._OPTIME: str(last_lsn)}, patch=True, retry=False)
+
     def _update_leader_with_retry(self, annotations: Dict[str, Any],
                                   resource_version: Optional[str], ips: List[str]) -> bool:
         retry = self._retry.copy()
@@ -1219,11 +1260,10 @@ class Kubernetes(AbstractDCS):
             else:
                 annotations['acquireTime'] = self._leader_observed_record.get('acquireTime') or now
             annotations['transitions'] = str(transitions)
-        ips: Optional[List[str]] = [] if self._api.use_endpoints else None
 
         try:
             ret = bool(self._patch_or_create(self.leader_path, annotations,
-                                             self._leader_resource_version, retry=self.retry, ips=ips))
+                                             self._leader_resource_version, retry=self.retry, ips=self.__ips))
         except k8s_client.rest.ApiException as e:
             if e.status == 409 and self._leader_resource_version:  # Conflict in resource_version
                 # Terminate watchers, it could be a sign that K8s API is in a failed state
@@ -1263,19 +1303,27 @@ class Kubernetes(AbstractDCS):
     def touch_member(self, data: Dict[str, Any]) -> bool:
         cluster = self.cluster
         if cluster and cluster.leader and cluster.leader.name == self._name:
-            role = 'master'
+            role = self._standby_leader_label_value if data['role'] == 'standby_leader' else self._leader_label_value
+            tmp_role = 'master'
         elif data['state'] == 'running' and data['role'] not in ('master', 'primary'):
-            role = data['role']
+            role = {'replica': self._follower_label_value}.get(data['role'], data['role'])
+            tmp_role = data['role']
         else:
             role = None
+            tmp_role = None
+
+        role_labels = {self._role_label: role}
+        if self._tmp_role_label:
+            role_labels[self._tmp_role_label] = tmp_role
 
         member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
         pod_labels = member and member.data.pop('pod_labels', None)
         ret = member and pod_labels is not None\
-            and pod_labels.get(self._role_label) == role and deep_compare(data, member.data)
+            and all(pod_labels.get(k) == v for k, v in role_labels.items())\
+            and deep_compare(data, member.data)
 
         if not ret:
-            metadata = {'namespace': self._namespace, 'name': self._name, 'labels': {self._role_label: role},
+            metadata = {'namespace': self._namespace, 'name': self._name, 'labels': role_labels,
                         'annotations': {'status': json.dumps(data, separators=(',', ':'))}}
             body = k8s_client.V1Pod(metadata=k8s_client.V1ObjectMeta(**metadata))
             ret = self._api.patch_namespaced_pod(self._name, self._namespace, body)
@@ -1291,11 +1339,11 @@ class Kubernetes(AbstractDCS):
             if cluster and cluster.config and cluster.config.version else None
         return self.patch_or_create_config({self._INITIALIZE: sysid}, resource_version)
 
-    def _delete_leader(self) -> bool:
+    def _delete_leader(self, leader: Leader) -> bool:
         """Unused"""
         raise NotImplementedError  # pragma: no cover
 
-    def delete_leader(self, last_lsn: Optional[int] = None) -> bool:
+    def delete_leader(self, leader: Optional[Leader], last_lsn: Optional[int] = None) -> bool:
         ret = False
         kind = self._kinds.get(self.leader_path)
         if kind and (kind.metadata.annotations or {}).get(self._LEADER) == self._name:
@@ -1322,15 +1370,18 @@ class Kubernetes(AbstractDCS):
         raise NotImplementedError  # pragma: no cover
 
     def write_sync_state(self, leader: Optional[str], sync_standby: Optional[Collection[str]],
-                         version: Optional[str] = None) -> Optional[SyncState]:
+                         quorum: Optional[int], version: Optional[str] = None) -> Optional[SyncState]:
         """Prepare and write annotations to $SCOPE-sync Endpoint or ConfigMap.
 
         :param leader: name of the leader node that manages /sync key
         :param sync_standby: collection of currently known synchronous standby node names
+        :param quorum: if the node from sync_standby list is doing a leader race it should
+                       see at least quorum other nodes from the sync_standby + leader list
         :param version: last known `resource_version` for conditional update of the object
         :returns: the new :class:`SyncState` object or None
         """
-        sync_state = self.sync_state(leader, sync_standby)
+        sync_state = self.sync_state(leader, sync_standby, quorum)
+        sync_state['quorum'] = str(sync_state['quorum']) if sync_state['quorum'] is not None else None
         ret = self.patch_or_create(self.sync_path, sync_state, version, False)
         if not isinstance(ret, bool):
             return SyncState.from_node(ret.metadata.resource_version, sync_state)
@@ -1342,7 +1393,7 @@ class Kubernetes(AbstractDCS):
         :param version: last known `resource_version` for conditional update of the object
         :returns: `True` if "delete" was successful
         """
-        return self.write_sync_state(None, None, version=version) is not None
+        return self.write_sync_state(None, None, None, version=version) is not None
 
     def watch(self, leader_version: Optional[str], timeout: float) -> bool:
         if self.__do_not_watch:

patroni/dcs/raft.py

@@ -12,8 +12,9 @@ from pysyncobj.transport import TCPTransport, CONNECTION_STATE
 from pysyncobj.utility import TcpUtility
 from typing import Any, Callable, Collection, Dict, List, Optional, Set, Union, TYPE_CHECKING
 
-from . import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory, citus_group_re
+from . import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, Status, SyncState, TimelineHistory
 from ..exceptions import DCSError
+from ..postgresql.mpp import AbstractMPP
 from ..utils import validate_directory
 if TYPE_CHECKING:  # pragma: no cover
     from ..config import Config
@@ -284,8 +285,8 @@ class KVStoreTTL(DynMemberSyncObj):
 
 class Raft(AbstractDCS):
 
-    def __init__(self, config: Dict[str, Any]) -> None:
+    def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None:
-        super(Raft, self).__init__(config)
+        super(Raft, self).__init__(config, mpp)
         self._ttl = int(config.get('ttl') or 30)
 
         ready_event = threading.Event()
@@ -343,23 +344,8 @@ class Raft(AbstractDCS):
         history = history and TimelineHistory.from_node(history['index'], history['value'])
 
         # get last know leader lsn and slots
-        status = nodes.get(self._STATUS)
+        status = nodes.get(self._STATUS) or nodes.get(self._LEADER_OPTIME)
-        if status:
+        status = Status.from_node(status and status['value'])
-            try:
-                status = json.loads(status['value'])
-                last_lsn = status.get(self._OPTIME)
-                slots = status.get('slots')
-            except Exception:
-                slots = last_lsn = None
-        else:
-            last_lsn = nodes.get(self._LEADER_OPTIME)
-            last_lsn = last_lsn and last_lsn['value']
-            slots = None
-
-        try:
-            last_lsn = int(last_lsn or '')
-        except Exception:
-            last_lsn = 0
 
         # get list of members
         members = [self.member(k, n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
@@ -387,21 +373,33 @@ class Raft(AbstractDCS):
         except Exception:
             failsafe = None
 
-        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
+        return Cluster(initialize, config, leader, status, members, failover, sync, history, failsafe)
 
-    def _cluster_loader(self, path: str) -> Cluster:
+    def _postgresql_cluster_loader(self, path: str) -> Cluster:
+        """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster.
+
+        :param path: the path in DCS where to load :class:`Cluster` from.
+
+        :returns: :class:`Cluster` instance.
+        """
         response = self._sync_obj.get(path, recursive=True)
         if not response:
             return Cluster.empty()
         nodes = {key[len(path):]: value for key, value in response.items()}
         return self._cluster_from_nodes(nodes)
 
-    def _citus_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+    def _mpp_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+        """Load and build all PostgreSQL clusters from a single MPP cluster.
+
+        :param path: the path in DCS where to load Cluster(s) from.
+
+        :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values.
+        """
         clusters: Dict[int, Dict[str, Any]] = defaultdict(dict)
         response = self._sync_obj.get(path, recursive=True)
         for key, value in (response or {}).items():
             key = key[len(path):].split('/', 1)
-            if len(key) == 2 and citus_group_re.match(key[0]):
+            if len(key) == 2 and self._mpp.group_re.match(key[0]):
                 clusters[int(key[0])][key[1]] = value
         return {group: self._cluster_from_nodes(nodes) for group, nodes in clusters.items()}
 
@@ -446,7 +444,7 @@ class Raft(AbstractDCS):
     def initialize(self, create_new: bool = True, sysid: str = '') -> bool:
         return self._sync_obj.set(self.initialize_path, sysid, prevExist=(not create_new)) is not False
 
-    def _delete_leader(self) -> bool:
+    def _delete_leader(self, leader: Leader) -> bool:
         return self._sync_obj.delete(self.leader_path, prevValue=self._name, timeout=1)
 
     def cancel_initialization(self) -> bool:

patroni/dcs/zookeeper.py

@@ -12,8 +12,9 @@ from kazoo.retry import RetryFailedError
 from kazoo.security import ACL, make_acl
 from typing import Any, Callable, Dict, List, Optional, Union, Tuple, TYPE_CHECKING
 
-from . import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory, citus_group_re
+from . import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, Status, SyncState, TimelineHistory
 from ..exceptions import DCSError
+from ..postgresql.mpp import AbstractMPP
 from ..utils import deep_compare
 if TYPE_CHECKING:  # pragma: no cover
     from ..config import Config
@@ -86,10 +87,10 @@ class PatroniKazooClient(KazooClient):
 
 class ZooKeeper(AbstractDCS):
 
-    def __init__(self, config: Dict[str, Any]) -> None:
+    def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None:
-        super(ZooKeeper, self).__init__(config)
+        super(ZooKeeper, self).__init__(config, mpp)
 
-        hosts = config.get('hosts', [])
+        hosts: Union[str, List[str]] = config.get('hosts', [])
         if isinstance(hosts, list):
             hosts = ','.join(hosts)
 
@@ -114,11 +115,9 @@ class ZooKeeper(AbstractDCS):
         self._client = PatroniKazooClient(hosts, handler=PatroniSequentialThreadingHandler(config['retry_timeout']),
                                           timeout=config['ttl'], connection_retry=KazooRetry(max_delay=1, max_tries=-1,
                                           sleep_func=time.sleep), command_retry=KazooRetry(max_delay=1, max_tries=-1,
-                                          deadline=config['retry_timeout'], sleep_func=time.sleep), **kwargs)
+                                          deadline=config['retry_timeout'], sleep_func=time.sleep),
-        self._client.add_listener(self.session_listener)
+                                          auth_data=list(config.get('auth_data', {}).items()), **kwargs)
 
-        self._fetch_cluster: bool = True
-        self._fetch_status: bool = True
         self.__last_member_data: Optional[Dict[str, Any]] = None
 
         self._orig_kazoo_connect = self._client._connection._connect
@@ -141,18 +140,9 @@ class ZooKeeper(AbstractDCS):
         ret = self._orig_kazoo_connect(*args)
         return max(self.loop_wait - 2, 2) * 1000, ret[1]
 
-    def session_listener(self, state: str) -> None:
+    def _watcher(self, event: WatchedEvent) -> None:
-        if state in [KazooState.SUSPENDED, KazooState.LOST]:
+        if event.state != KazooState.CONNECTED or event.path.startswith(self.client_path('')):
-            self.cluster_watcher(None)
+            self.event.set()
-
-    def status_watcher(self, event: Optional[WatchedEvent]) -> None:
-        self._fetch_status = True
-        self.event.set()
-
-    def cluster_watcher(self, event: Optional[WatchedEvent]) -> None:
-        self._fetch_cluster = True
-        if not event or event.state != KazooState.CONNECTED or event.path.startswith(self.client_path('')):
-            self.status_watcher(event)
 
     def reload_config(self, config: Union['Config', Dict[str, Any]]) -> None:
         self.set_retry_timeout(config['retry_timeout'])
@@ -200,138 +190,101 @@ class ZooKeeper(AbstractDCS):
         except NoNodeError:
             return None
 
-    def get_status(self, path: str, leader: Optional[Leader]) -> Tuple[int, Optional[Dict[str, int]]]:
+    def get_status(self, path: str, leader: Optional[Leader]) -> Status:
-        watch = self.status_watcher if not leader or leader.name != self._name else None
+        status = self.get_node(path + self._STATUS)
-
+        if not status:
-        status = self.get_node(path + self._STATUS, watch)
+            status = self.get_node(path + self._LEADER_OPTIME)
-        if status:
+        return Status.from_node(status and status[0])
-            try:
-                status = json.loads(status[0])
-                last_lsn = status.get(self._OPTIME)
-                slots = status.get('slots')
-            except Exception:
-                slots = last_lsn = None
-        else:
-            last_lsn = self.get_node(path + self._LEADER_OPTIME, watch)
-            last_lsn = last_lsn and last_lsn[0]
-            slots = None
-
-        try:
-            last_lsn = int(last_lsn or '')
-        except Exception:
-            last_lsn = 0
-
-        self._fetch_status = False
-        return last_lsn, slots
 
     @staticmethod
     def member(name: str, value: str, znode: ZnodeStat) -> Member:
         return Member.from_node(znode.version, name, znode.ephemeralOwner, value)
 
-    def get_children(self, key: str, watch: Optional[Callable[[WatchedEvent], None]] = None) -> List[str]:
+    def get_children(self, key: str) -> List[str]:
         try:
-            return self._client.get_children(key, watch)
+            return self._client.get_children(key)
         except NoNodeError:
             return []
 
     def load_members(self, path: str) -> List[Member]:
         members: List[Member] = []
-        for member in self.get_children(path + self._MEMBERS, self.cluster_watcher):
+        for member in self.get_children(path + self._MEMBERS):
             data = self.get_node(path + self._MEMBERS + member)
             if data is not None:
                 members.append(self.member(member, *data))
         return members
 
-    def _cluster_loader(self, path: str) -> Cluster:
+    def _postgresql_cluster_loader(self, path: str) -> Cluster:
-        self._fetch_cluster = False
+        """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster.
-        self.event.clear()
+
-        nodes = set(self.get_children(path, self.cluster_watcher))
+        :param path: the path in DCS where to load :class:`Cluster` from.
-        if not nodes:
+
-            self._fetch_cluster = True
+        :returns: :class:`Cluster` instance.
+        """
+        nodes = set(self.get_children(path))
 
         # get initialize flag
         initialize = (self.get_node(path + self._INITIALIZE) or [None])[0] if self._INITIALIZE in nodes else None
 
         # get global dynamic configuration
-        config = self.get_node(path + self._CONFIG, watch=self.cluster_watcher) if self._CONFIG in nodes else None
+        config = self.get_node(path + self._CONFIG, watch=self._watcher) if self._CONFIG in nodes else None
         config = config and ClusterConfig.from_node(config[1].version, config[0], config[1].mzxid)
 
         # get timeline history
-        history = self.get_node(path + self._HISTORY, watch=self.cluster_watcher) if self._HISTORY in nodes else None
+        history = self.get_node(path + self._HISTORY) if self._HISTORY in nodes else None
         history = history and TimelineHistory.from_node(history[1].mzxid, history[0])
 
         # get synchronization state
-        sync = self.get_node(path + self._SYNC, watch=self.cluster_watcher) if self._SYNC in nodes else None
+        sync = self.get_node(path + self._SYNC) if self._SYNC in nodes else None
         sync = SyncState.from_node(sync and sync[1].version, sync and sync[0])
 
         # get list of members
         members = self.load_members(path) if self._MEMBERS[:-1] in nodes else []
 
         # get leader
-        leader = self.get_node(path + self._LEADER) if self._LEADER in nodes else None
+        leader = self.get_node(path + self._LEADER, watch=self._watcher) if self._LEADER in nodes else None
         if leader:
             member = Member(-1, leader[0], None, {})
             member = ([m for m in members if m.name == leader[0]] or [member])[0]
             leader = Leader(leader[1].version, leader[1].ephemeralOwner, member)
-            self._fetch_cluster = member.version == -1
 
         # get last known leader lsn and slots
-        last_lsn, slots = self.get_status(path, leader)
+        status = self.get_status(path, leader)
 
         # failover key
-        failover = self.get_node(path + self._FAILOVER, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
+        failover = self.get_node(path + self._FAILOVER) if self._FAILOVER in nodes else None
         failover = failover and Failover.from_node(failover[1].version, failover[0])
 
         # get failsafe topology
-        failsafe = self.get_node(path + self._FAILSAFE, watch=self.cluster_watcher) if self._FAILSAFE in nodes else None
+        failsafe = self.get_node(path + self._FAILSAFE) if self._FAILSAFE in nodes else None
         try:
             failsafe = json.loads(failsafe[0]) if failsafe else None
         except Exception:
             failsafe = None
 
-        return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
+        return Cluster(initialize, config, leader, status, members, failover, sync, history, failsafe)
 
-    def _citus_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+    def _mpp_cluster_loader(self, path: str) -> Dict[int, Cluster]:
-        fetch_cluster = False
+        """Load and build all PostgreSQL clusters from a single MPP cluster.
+
+        :param path: the path in DCS where to load Cluster(s) from.
+
+        :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values.
+        """
         ret: Dict[int, Cluster] = {}
-        for node in self.get_children(path, self.cluster_watcher):
+        for node in self.get_children(path):
-            if citus_group_re.match(node):
+            if self._mpp.group_re.match(node):
-                ret[int(node)] = self._cluster_loader(path + node + '/')
+                ret[int(node)] = self._postgresql_cluster_loader(path + node + '/')
-                fetch_cluster = fetch_cluster or self._fetch_cluster
-        self._fetch_cluster = fetch_cluster
         return ret
 
     def _load_cluster(
             self, path: str, loader: Callable[[str], Union[Cluster, Dict[int, Cluster]]]
     ) -> Union[Cluster, Dict[int, Cluster]]:
-        cluster = self.cluster if path == self._base_path + '/' else None
+        try:
-        if self._fetch_cluster or cluster is None:
+            return self._client.retry(loader, path)
-            try:
+        except Exception:
-                cluster = self._client.retry(loader, path)
+            logger.exception('get_cluster')
-            except Exception:
+            raise ZooKeeperError('ZooKeeper in not responding properly')
-                logger.exception('get_cluster')
-                self.cluster_watcher(None)
-                raise ZooKeeperError('ZooKeeper in not responding properly')
-        # The /status ZNode was updated or doesn't exist
-        elif self._fetch_status and not self._fetch_cluster or not cluster.last_lsn \
-                or cluster.has_permanent_logical_slots(self._name, False) and not cluster.slots:
-            # If current node is the leader just clear the event without fetching anything (we are updating the /status)
-            if cluster.leader and cluster.leader.name == self._name:
-                self.event.clear()
-            else:
-                try:
-                    last_lsn, slots = self.get_status(self.client_path(''), cluster.leader)
-                    self.event.clear()
-                    new_cluster: List[Any] = list(cluster)
-                    new_cluster[3] = last_lsn
-                    new_cluster[8] = slots
-                    cluster = Cluster(*new_cluster)
-                except Exception:
-                    pass
-        return cluster
-
-    def _bypass_caches(self) -> None:
-        self._fetch_cluster = True
 
     def _create(self, path: str, value: bytes, retry: bool = False, ephemeral: bool = False) -> bool:
         try:
@@ -393,21 +346,17 @@ class ZooKeeper(AbstractDCS):
         cluster = self.cluster
         member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
         member_data = self.__last_member_data or member and member.data
-        #  We want to notify leader if some important fields in the member key changed by removing ZNode
+        if member and member_data:
-        if member and (self._client.client_id is not None and member.session != self._client.client_id[0]
+            # We want delete the member ZNode if our session doesn't match with session id on our member key
-                       or not (member_data and deep_compare(member_data.get('tags', {}), data.get('tags', {}))
+            if self._client.client_id is not None and member.session != self._client.client_id[0]:
-                               and (member_data.get('state') == data.get('state')
+                logger.warning('Recreating the member ZNode due to ownership mismatch')
-                                    or 'running' not in (member_data.get('state'), data.get('state')))
+                try:
-                               and member_data.get('version') == data.get('version')
+                    self._client.delete_async(self.member_path).get(timeout=1)
-                               and member_data.get('checkpoint_after_promote')
+                except NoNodeError:
-                               == data.get('checkpoint_after_promote'))):
+                    pass
-            try:
+                except Exception:
-                self._client.delete_async(self.member_path).get(timeout=1)
+                    return False
-            except NoNodeError:
+                member = None
-                pass
-            except Exception:
-                return False
-            member = None
 
         encoded_data = json.dumps(data, separators=(',', ':')).encode('utf-8')
         if member and member_data:
@@ -466,7 +415,7 @@ class ZooKeeper(AbstractDCS):
                 return False
         return True
 
-    def _delete_leader(self) -> bool:
+    def _delete_leader(self, leader: Leader) -> bool:
         self._client.restart()
         return True
 
@@ -499,7 +448,10 @@ class ZooKeeper(AbstractDCS):
         return self.set_sync_state_value("{}", version) is not False
 
     def watch(self, leader_version: Optional[int], timeout: float) -> bool:
-        ret = super(ZooKeeper, self).watch(leader_version, timeout + 0.5)
+        if leader_version:
-        if ret and not self._fetch_status:
+            timeout += 0.5
-            self._fetch_cluster = True
+
-        return ret or self._fetch_cluster
+        try:
+            return super(ZooKeeper, self).watch(leader_version, timeout)
+        finally:
+            self.event.clear()

patroni/dynamic_loader.py

@@ -0,0 +1,96 @@
+"""Helper functions to search for implementations of specific abstract interface in a package."""
+import importlib
+import inspect
+import logging
+import os
+import pkgutil
+import sys
+from types import ModuleType
+
+from typing import Any, Dict, Iterator, List, Optional, Set, Tuple, TYPE_CHECKING, Type, TypeVar, Union
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .config import Config
+
+logger = logging.getLogger(__name__)
+
+
+def iter_modules(package: str) -> List[str]:
+    """Get names of modules from *package*, depending on execution environment.
+
+    .. note::
+        If being packaged with PyInstaller, modules aren't discoverable dynamically by scanning source directory because
+        :class:`importlib.machinery.FrozenImporter` doesn't implement :func:`iter_modules`. But it is still possible to
+        find all potential modules by iterating through ``toc``, which contains list of all "frozen" resources.
+
+    :param package: a package name to search modules in, e.g. ``patroni.dcs``.
+
+    :returns: list of known module names with absolute python module path namespace, e.g. ``patroni.dcs.etcd``.
+    """
+    module_prefix = package + '.'
+
+    if getattr(sys, 'frozen', False):
+        toc: Set[str] = set()
+        # dirname may contain a few dots, which causes pkgutil.iter_importers()
+        # to misinterpret the path as a package name. This can be avoided
+        # altogether by not passing a path at all, because PyInstaller's
+        # FrozenImporter is a singleton and registered as top-level finder.
+        for importer in pkgutil.iter_importers():
+            if hasattr(importer, 'toc'):
+                toc |= getattr(importer, 'toc')
+        dots = module_prefix.count('.')  # search for modules only on the same level
+        return [module for module in toc if module.startswith(module_prefix) and module.count('.') == dots]
+
+    # here we are making an assumption that the package which is calling this function is already imported
+    pkg_file = sys.modules[package].__file__
+    if TYPE_CHECKING:  # pragma: no cover
+        assert isinstance(pkg_file, str)
+    return [name for _, name, is_pkg in pkgutil.iter_modules([os.path.dirname(pkg_file)], module_prefix) if not is_pkg]
+
+
+ClassType = TypeVar("ClassType")
+
+
+def find_class_in_module(module: ModuleType, cls_type: Type[ClassType]) -> Optional[Type[ClassType]]:
+    """Try to find the implementation of *cls_type* class interface in *module* matching the *module* name.
+
+    :param module: imported module.
+    :param cls_type: a class type we are looking for.
+
+    :returns: class with a name matching the name of *module* that implements *cls_type* or ``None`` if not found.
+    """
+    module_name = module.__name__.rpartition('.')[2]
+    return next(
+        (obj for obj_name, obj in module.__dict__.items()
+         if (obj_name.lower() == module_name
+             and inspect.isclass(obj) and issubclass(obj, cls_type))),
+        None)
+
+
+def iter_classes(
+        package: str, cls_type: Type[ClassType],
+        config: Optional[Union['Config', Dict[str, Any]]] = None
+) -> Iterator[Tuple[str, Type[ClassType]]]:
+    """Attempt to import modules and find implementations of *cls_type* that are present in the given configuration.
+
+    .. note::
+            If a module successfully imports we can assume that all its requirements are installed.
+
+    :param package: a package name to search modules in, e.g. ``patroni.dcs``.
+    :param cls_type: a class type we are looking for.
+    :param config: configuration information with possible module names as keys. If given, only attempt to import
+                   modules defined in the configuration. Else, if ``None``, attempt to import any supported module.
+
+    :yields: a tuple containing the module ``name`` and the imported class object.
+    """
+    for mod_name in iter_modules(package):
+        name = mod_name.rpartition('.')[2]
+        if config is None or name in config:
+            try:
+                module = importlib.import_module(mod_name)
+                module_cls = find_class_in_module(module, cls_type)
+                if module_cls:
+                    yield name, module_cls
+            except ImportError:
+                logger.log(logging.DEBUG if config is not None else logging.INFO,
+                           'Failed to import %s', mod_name)

patroni/file_perm.py

@@ -0,0 +1,95 @@
+"""Helper object that helps with figuring out file and directory permissions based on permissions of PGDATA.
+
+:var logger: logger of this module.
+:var pg_perm: instance of the :class:`__FilePermissions` object.
+"""
+import logging
+import os
+import stat
+
+logger = logging.getLogger(__name__)
+
+
+class __FilePermissions:
+    """Helper class for managing permissions of directories and files under PGDATA.
+
+    Execute :meth:`set_permissions_from_data_directory` to figure out which permissions should be used for files and
+    directories under PGDATA based on permissions of PGDATA root directory.
+    """
+
+    # Mode mask for data directory permissions that only allows the owner to
+    # read/write directories and files -- mask 077.
+    __PG_MODE_MASK_OWNER = stat.S_IRWXG | stat.S_IRWXO
+
+    # Mode mask for data directory permissions that also allows group read/execute -- mask 027.
+    __PG_MODE_MASK_GROUP = stat.S_IWGRP | stat.S_IRWXO
+
+    # Default mode for creating directories -- mode 700.
+    __PG_DIR_MODE_OWNER = stat.S_IRWXU
+
+    # Mode for creating directories that allows group read/execute -- mode 750.
+    __PG_DIR_MODE_GROUP = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP
+
+    # Default mode for creating files -- mode 600.
+    __PG_FILE_MODE_OWNER = stat.S_IRUSR | stat.S_IWUSR
+
+    # Mode for creating files that allows group read -- mode 640.
+    __PG_FILE_MODE_GROUP = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP
+
+    def __init__(self) -> None:
+        """Create a :class:`__FilePermissions` object and set default permissions."""
+        self.__set_owner_permissions()
+        self.__set_umask()
+
+    def __set_umask(self) -> None:
+        """Set umask value based on calculations.
+
+        .. note::
+            Should only be called once either :meth:`__set_owner_permissions`
+            or :meth:`__set_group_permissions` has been executed.
+        """
+        try:
+            os.umask(self.__pg_mode_mask)
+        except Exception as e:
+            logger.error('Can not set umask to %03o: %r', self.__pg_mode_mask, e)
+
+    def __set_owner_permissions(self) -> None:
+        """Make directories/files accessible only by the owner."""
+        self.__pg_dir_create_mode = self.__PG_DIR_MODE_OWNER
+        self.__pg_file_create_mode = self.__PG_FILE_MODE_OWNER
+        self.__pg_mode_mask = self.__PG_MODE_MASK_OWNER
+
+    def __set_group_permissions(self) -> None:
+        """Make directories/files accessible by the owner and readable by group."""
+        self.__pg_dir_create_mode = self.__PG_DIR_MODE_GROUP
+        self.__pg_file_create_mode = self.__PG_FILE_MODE_GROUP
+        self.__pg_mode_mask = self.__PG_MODE_MASK_GROUP
+
+    def set_permissions_from_data_directory(self, data_dir: str) -> None:
+        """Set new permissions based on provided *data_dir*.
+
+        :param data_dir: reference to PGDATA to calculate permissions from.
+        """
+        try:
+            st = os.stat(data_dir)
+            if (st.st_mode & self.__PG_DIR_MODE_GROUP) == self.__PG_DIR_MODE_GROUP:
+                self.__set_group_permissions()
+            else:
+                self.__set_owner_permissions()
+        except Exception as e:
+            logger.error('Can not check permissions on %s: %r', data_dir, e)
+        else:
+            self.__set_umask()
+
+    @property
+    def dir_create_mode(self) -> int:
+        """Directory permissions."""
+        return self.__pg_dir_create_mode
+
+    @property
+    def file_create_mode(self) -> int:
+        """File permissions."""
+        return self.__pg_file_create_mode
+
+
+pg_perm = __FilePermissions()

patroni/global_config.py

@@ -0,0 +1,234 @@
+"""Implements *global_config* facilities.
+
+The :class:`GlobalConfig` object is instantiated on import and replaces
+``patroni.global_config`` module in :data:`sys.modules`, what allows to use
+its properties and methods like they were module variables and functions.
+"""
+import sys
+import types
+
+from copy import deepcopy
+from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING
+
+from .utils import parse_bool, parse_int
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .dcs import Cluster
+
+
+def __getattr__(mod: types.ModuleType, name: str) -> Any:
+    """This function exists just to make pyright happy.
+
+    Without it pyright complains about access to unknown members of global_config module.
+    """
+    return getattr(sys.modules[__name__], name)  # pragma: no cover
+
+
+class GlobalConfig(types.ModuleType):
+    """A class that wraps global configuration and provides convenient methods to access/check values."""
+
+    __file__ = __file__  # just to make unittest and pytest happy
+
+    def __init__(self) -> None:
+        """Initialize :class:`GlobalConfig` object."""
+        super().__init__(__name__)
+        self.__config = {}
+
+    @staticmethod
+    def _cluster_has_valid_config(cluster: Optional['Cluster']) -> bool:
+        """Check if provided *cluster* object has a valid global configuration.
+
+        :param cluster: the currently known cluster state from DCS.
+
+        :returns: ``True`` if provided *cluster* object has a valid global configuration, otherwise ``False``.
+        """
+        return bool(cluster and cluster.config and cluster.config.modify_version)
+
+    def update(self, cluster: Optional['Cluster'], default: Optional[Dict[str, Any]] = None) -> None:
+        """Update with the new global configuration from the :class:`Cluster` object view.
+
+        .. note::
+            Update happens in-place and is executed only from the main heartbeat thread.
+
+        :param cluster: the currently known cluster state from DCS.
+        :param default: default configuration, which will be used if there is no valid *cluster.config*.
+        """
+        # Try to protect from the case when DCS was wiped out
+        if self._cluster_has_valid_config(cluster):
+            self.__config = cluster.config.data  # pyright: ignore [reportOptionalMemberAccess]
+        elif default:
+            self.__config = default
+
+    def from_cluster(self, cluster: Optional['Cluster']) -> 'GlobalConfig':
+        """Return :class:`GlobalConfig` instance from the provided :class:`Cluster` object view.
+
+        .. note::
+            If the provided *cluster* object doesn't have a valid global configuration we return
+            the last known valid state of the :class:`GlobalConfig` object.
+
+            This method is used when we need to have the most up-to-date values in the global configuration,
+            but we don't want to update the global object.
+
+        :param cluster: the currently known cluster state from DCS.
+
+        :returns: :class:`GlobalConfig` object.
+        """
+        if not self._cluster_has_valid_config(cluster):
+            return self
+
+        ret = GlobalConfig()
+        ret.update(cluster)
+        return ret
+
+    def get(self, name: str) -> Any:
+        """Gets global configuration value by *name*.
+
+        :param name: parameter name.
+
+        :returns: configuration value or ``None`` if it is missing.
+        """
+        return self.__config.get(name)
+
+    def check_mode(self, mode: str) -> bool:
+        """Checks whether the certain parameter is enabled.
+
+        :param mode: parameter name, e.g. ``synchronous_mode``, ``failsafe_mode``, ``pause``, ``check_timeline``, and
+            so on.
+
+        :returns: ``True`` if parameter *mode* is enabled in the global configuration.
+        """
+        return bool(parse_bool(self.__config.get(mode)))
+
+    @property
+    def is_paused(self) -> bool:
+        """``True`` if cluster is in maintenance mode."""
+        return self.check_mode('pause')
+
+    @property
+    def is_quorum_commit_mode(self) -> bool:
+        """:returns: ``True`` if quorum commit replication is requested"""
+        return str(self.get('synchronous_mode')).lower() == 'quorum'
+
+    @property
+    def is_synchronous_mode(self) -> bool:
+        """``True`` if synchronous replication is requested and it is not a standby cluster config."""
+        return (self.check_mode('synchronous_mode') is True or self.is_quorum_commit_mode) \
+            and not self.is_standby_cluster
+
+    @property
+    def is_synchronous_mode_strict(self) -> bool:
+        """``True`` if at least one synchronous node is required."""
+        return self.check_mode('synchronous_mode_strict')
+
+    def get_standby_cluster_config(self) -> Union[Dict[str, Any], Any]:
+        """Get ``standby_cluster`` configuration.
+
+        :returns: a copy of ``standby_cluster`` configuration.
+        """
+        return deepcopy(self.get('standby_cluster'))
+
+    @property
+    def is_standby_cluster(self) -> bool:
+        """``True`` if global configuration has a valid ``standby_cluster`` section."""
+        config = self.get_standby_cluster_config()
+        return isinstance(config, dict) and\
+            bool(config.get('host') or config.get('port') or config.get('restore_command'))
+
+    def get_int(self, name: str, default: int = 0) -> int:
+        """Gets current value of *name* from the global configuration and try to return it as :class:`int`.
+
+        :param name: name of the parameter.
+        :param default: default value if *name* is not in the configuration or invalid.
+
+        :returns: currently configured value of *name* from the global configuration or *default* if it is not set or
+            invalid.
+        """
+        ret = parse_int(self.get(name))
+        return default if ret is None else ret
+
+    @property
+    def min_synchronous_nodes(self) -> int:
+        """The minimum number of synchronous nodes based on whether ``synchronous_mode_strict`` is enabled or not."""
+        return 1 if self.is_synchronous_mode_strict else 0
+
+    @property
+    def synchronous_node_count(self) -> int:
+        """Currently configured value of ``synchronous_node_count`` from the global configuration.
+
+        Assume ``1`` if it is not set or invalid.
+        """
+        return max(self.get_int('synchronous_node_count', 1), self.min_synchronous_nodes)
+
+    @property
+    def maximum_lag_on_failover(self) -> int:
+        """Currently configured value of ``maximum_lag_on_failover`` from the global configuration.
+
+        Assume ``1048576`` if it is not set or invalid.
+        """
+        return self.get_int('maximum_lag_on_failover', 1048576)
+
+    @property
+    def maximum_lag_on_syncnode(self) -> int:
+        """Currently configured value of ``maximum_lag_on_syncnode`` from the global configuration.
+
+        Assume ``-1`` if it is not set or invalid.
+        """
+        return self.get_int('maximum_lag_on_syncnode', -1)
+
+    @property
+    def primary_start_timeout(self) -> int:
+        """Currently configured value of ``primary_start_timeout`` from the global configuration.
+
+        Assume ``300`` if it is not set or invalid.
+
+        .. note::
+            ``master_start_timeout`` is still supported to keep backward compatibility.
+        """
+        default = 300
+        return self.get_int('primary_start_timeout', default)\
+            if 'primary_start_timeout' in self.__config else self.get_int('master_start_timeout', default)
+
+    @property
+    def primary_stop_timeout(self) -> int:
+        """Currently configured value of ``primary_stop_timeout`` from the global configuration.
+
+        Assume ``0`` if it is not set or invalid.
+
+        .. note::
+            ``master_stop_timeout`` is still supported to keep backward compatibility.
+        """
+        default = 0
+        return self.get_int('primary_stop_timeout', default)\
+            if 'primary_stop_timeout' in self.__config else self.get_int('master_stop_timeout', default)
+
+    @property
+    def ignore_slots_matchers(self) -> List[Dict[str, Any]]:
+        """Currently configured value of ``ignore_slots`` from the global configuration.
+
+        Assume an empty :class:`list` if not set.
+        """
+        return self.get('ignore_slots') or []
+
+    @property
+    def max_timelines_history(self) -> int:
+        """Currently configured value of ``max_timelines_history`` from the global configuration.
+
+        Assume ``0`` if not set or invalid.
+        """
+        return self.get_int('max_timelines_history', 0)
+
+    @property
+    def use_slots(self) -> bool:
+        """``True`` if cluster is configured to use replication slots."""
+        return bool(parse_bool((self.get('postgresql') or {}).get('use_slots', True)))
+
+    @property
+    def permanent_slots(self) -> Dict[str, Any]:
+        """Dictionary of permanent slots information from the global configuration."""
+        return deepcopy(self.get('permanent_replication_slots')
+                        or self.get('permanent_slots')
+                        or self.get('slots')
+                        or {})
+
+
+sys.modules[__name__] = GlobalConfig()

patroni/log.py

@@ -9,29 +9,32 @@ import sys
 
 from copy import deepcopy
 from logging.handlers import RotatingFileHandler
-from patroni.utils import deep_compare
 from queue import Queue, Full
 from threading import Lock, Thread
 
 from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING
 
+from .utils import deep_compare
+
+type_logformat = Union[List[Union[str, Dict[str, Any], Any]], str, Any]
+
 _LOGGER = logging.getLogger(__name__)
 
 
 def debug_exception(self: logging.Logger, msg: object, *args: Any, **kwargs: Any) -> None:
     """Add full stack trace info to debug log messages and partial to others.
 
-    Handle :func:`exception` calls for *self*.
+    Handle :func:`~self.exception` calls for *self*.
 
     .. note::
         * If *self* log level is set to ``DEBUG``, then issue a ``DEBUG`` message with the complete stack trace;
         * If *self* log level is ``INFO`` or higher, then issue an ``ERROR`` message with only the last line of
             the stack trace.
 
-    :param self: logger for which :func:`exception` will be processed.
+    :param self: logger for which :func:`~self.exception` will be processed.
     :param msg: the message related to the exception to be logged.
-    :param args: positional arguments to be passed to :func:`self.debug` or :func:`loger_obj.error`.
+    :param args: positional arguments to be passed to :func:`~self.debug` or :func:`~self.error`.
-    :param kwargs: keyword arguments to be passed to :func:`self.debug` or :func:`loger_obj.error`.
+    :param kwargs: keyword arguments to be passed to :func:`~self.debug` or :func:`~self.error`.
     """
     kwargs.pop("exc_info", False)
     if self.isEnabledFor(logging.DEBUG):
@@ -44,16 +47,16 @@ def debug_exception(self: logging.Logger, msg: object, *args: Any, **kwargs: Any
 def error_exception(self: logging.Logger, msg: object, *args: Any, **kwargs: Any) -> None:
     """Add full stack trace info to error messages.
 
-    Handle :func:`exception` calls for *self*.
+    Handle :func:`~self.exception` calls for *self*.
 
     .. note::
         * By default issue an ``ERROR`` message with the complete stack trace. If you do not want to show the complete
-        stack trace, call with ``exc_info=False``.
+          stack trace, call with ``exc_info=False``.
 
-    :param self: logger for which :func:`exception` will be processed.
+    :param self: logger for which :func:`~self.exception` will be processed.
     :param msg: the message related to the exception to be logged.
-    :param args: positional arguments to be passed to :func:`loger_obj.error`.
+    :param args: positional arguments to be passed to :func:`~self.error`.
-    :param kwargs: keyword arguments to be passed to :func:`loger_obj.error`.
+    :param kwargs: keyword arguments to be passed to :func:`~self.error`.
     """
     exc_info = kwargs.pop("exc_info", True)
     self.error(msg, *args, exc_info=exc_info, **kwargs)
@@ -140,7 +143,7 @@ class ProxyHandler(logging.Handler):
     def emit(self, record: logging.LogRecord) -> None:
         """Emit each log record that is handled.
 
-        Will push the log record down to :func:`handle` method of the currently configured log handler.
+        Will push the log record down to :func:`~logging.Handler.handle` method of the currently configured log handler.
 
         :param record: the record that was emitted.
         """
@@ -157,6 +160,7 @@ class PatroniLogger(Thread):
     .. seealso::
         :class:`QueueHandler`: object used for enqueueing messages in-memory.
 
+    :cvar DEFAULT_TYPE: default type of log format (``plain``).
     :cvar DEFAULT_LEVEL: default logging level (``INFO``).
     :cvar DEFAULT_TRACEBACK_LEVEL: default traceback logging level (``ERROR``).
     :cvar DEFAULT_FORMAT: default format of log messages (``%(asctime)s %(levelname)s: %(message)s``).
@@ -169,6 +173,7 @@ class PatroniLogger(Thread):
     :ivar log_handler_lock: lock used to modify ``log_handler``.
     """
 
+    DEFAULT_TYPE = 'plain'
     DEFAULT_LEVEL = 'INFO'
     DEFAULT_TRACEBACK_LEVEL = 'ERROR'
     DEFAULT_FORMAT = '%(asctime)s %(levelname)s: %(message)s'
@@ -202,28 +207,186 @@ class PatroniLogger(Thread):
         self._proxy_handler = ProxyHandler(self)
         self._root_logger.addHandler(self._proxy_handler)
 
-    def update_loggers(self) -> None:
+    def update_loggers(self, config: Dict[str, Any]) -> None:
-        """Configure loggers' log level as defined in ``log.loggers` section of Patroni configuration.
+        """Configure custom loggers' log levels.
 
         .. note::
             It creates logger objects that are not defined yet in the log manager.
+
+        :param config: :class:`dict` object with custom loggers configuration, is set either from:
+
+                       * ``log.loggers`` section of Patroni configuration; or
+
+                       * from the method that is trying to make sure that the node name
+                         isn't duplicated (to silence annoying ``urllib3`` WARNING's).
+
+        :Example:
+
+            .. code-block:: python
+
+                update_loggers({'urllib3.connectionpool': 'WARNING'})
         """
-        loggers = deepcopy((self._config or {}).get('loggers') or {})
+        loggers = deepcopy(config)
         for name, logger in self._root_logger.manager.loggerDict.items():
             # ``Placeholder`` is a node in the log manager for which no logger has been defined. We are interested only
             # in the ones that were defined
             if not isinstance(logger, logging.PlaceHolder):
-                # if this logger is present in ``log.loggers`` Patroni configuration, use the configured level,
+                # if this logger is present in *config*, use the configured level, otherwise
-                # otherwise use ``logging.NOTSET``, which means it will inherit the level from any parent node up to
+                # use ``logging.NOTSET``, which means it will inherit the level
-                # the root for which log level is defined.
+                # from any parent node up to the root for which log level is defined.
                 level = loggers.pop(name, logging.NOTSET)
                 logger.setLevel(level)
 
-        # define loggers that do not exist yet and set level as configured in ``log.loggers`` section of configuration.
+        # define loggers that do not exist yet and set level as configured in the *config*
         for name, level in loggers.items():
             logger = self._root_logger.manager.getLogger(name)
             logger.setLevel(level)
 
+    def _is_config_changed(self, config: Dict[str, Any]) -> bool:
+        """Checks if the given config is different from the current one.
+
+        :param config: ``log`` section from Patroni configuration.
+
+        :returns: ``True`` if the config is changed, ``False`` otherwise.
+        """
+        old_config = self._config or {}
+
+        oldlogtype = old_config.get('type', PatroniLogger.DEFAULT_TYPE)
+        logtype = config.get('type', PatroniLogger.DEFAULT_TYPE)
+
+        oldlogformat: type_logformat = old_config.get('format', PatroniLogger.DEFAULT_FORMAT)
+        logformat: type_logformat = config.get('format', PatroniLogger.DEFAULT_FORMAT)
+
+        olddateformat = old_config.get('dateformat') or None
+        dateformat = config.get('dateformat') or None  # Convert empty string to `None`
+
+        old_static_fields = old_config.get('static_fields', {})
+        static_fields = config.get('static_fields', {})
+
+        old_log_config = {
+            'type': oldlogtype,
+            'format': oldlogformat,
+            'dateformat': olddateformat,
+            'static_fields': old_static_fields
+        }
+
+        log_config = {
+            'type': logtype,
+            'format': logformat,
+            'dateformat': dateformat,
+            'static_fields': static_fields
+        }
+
+        return not deep_compare(old_log_config, log_config)
+
+    def _get_plain_formatter(self, logformat: type_logformat, dateformat: Optional[str]) -> logging.Formatter:
+        """Returns a logging formatter with the specified format and date format.
+
+        .. note::
+            If the log format isn't a string, prints a warning message and uses the default log format instead.
+
+        :param logformat: The format of the log messages.
+        :param dateformat: The format of the timestamp in the log messages.
+
+        :returns: A logging formatter object that can be used to format log records.
+        """
+
+        if not isinstance(logformat, str):
+            _LOGGER.warning('Expected log format to be a string when log type is plain, but got "%s"', type(logformat))
+            logformat = PatroniLogger.DEFAULT_FORMAT
+
+        return logging.Formatter(logformat, dateformat)
+
+    def _get_json_formatter(self, logformat: type_logformat, dateformat: Optional[str],
+                            static_fields: Dict[str, Any]) -> logging.Formatter:
+        """Returns a logging formatter that outputs JSON formatted messages.
+
+        .. note::
+            If :mod:`pythonjsonlogger` library is not installed, prints an error message and returns
+            a plain log formatter instead.
+
+        :param logformat: Specifies the log fields and their key names in the JSON log message.
+        :param dateformat: The format of the timestamp in the log messages.
+        :param static_fields: A dictionary of static fields that are added to every log message.
+
+        :returns: A logging formatter object that can be used to format log records as JSON strings.
+        """
+
+        if isinstance(logformat, str):
+            jsonformat = logformat
+            rename_fields = {}
+        elif isinstance(logformat, list):
+            log_fields: List[str] = []
+            rename_fields: Dict[str, str] = {}
+
+            for field in logformat:
+                if isinstance(field, str):
+                    log_fields.append(field)
+                elif isinstance(field, dict):
+                    for original_field, renamed_field in field.items():
+                        if isinstance(renamed_field, str):
+                            log_fields.append(original_field)
+                            rename_fields[original_field] = renamed_field
+                        else:
+                            _LOGGER.warning(
+                                'Expected renamed log field to be a string, but got "%s"',
+                                type(renamed_field)
+                            )
+
+                else:
+                    _LOGGER.warning(
+                        'Expected each item of log format to be a string or dictionary, but got "%s"',
+                        type(field)
+                    )
+
+            if len(log_fields) > 0:
+                jsonformat = ' '.join([f'%({field})s' for field in log_fields])
+            else:
+                jsonformat = PatroniLogger.DEFAULT_FORMAT
+        else:
+            jsonformat = PatroniLogger.DEFAULT_FORMAT
+            rename_fields = {}
+            _LOGGER.warning('Expected log format to be a string or a list, but got "%s"', type(logformat))
+
+        try:
+            from pythonjsonlogger import jsonlogger
+
+            return jsonlogger.JsonFormatter(
+                jsonformat,
+                dateformat,
+                rename_fields=rename_fields,
+                static_fields=static_fields
+            )
+        except ImportError as e:
+            _LOGGER.error('Failed to import "python-json-logger" library: %r. Falling back to the plain logger', e)
+        except Exception as e:
+            _LOGGER.error('Failed to initialize JsonFormatter: %r. Falling back to the plain logger', e)
+
+        return self._get_plain_formatter(jsonformat, dateformat)
+
+    def _get_formatter(self, config: Dict[str, Any]) -> logging.Formatter:
+        """Returns a logging formatter based on the type of logger in the given configuration.
+
+        :param config: ``log`` section from Patroni configuration.
+
+        :returns: A :class:`logging.Formatter` object that can be used to format log records.
+        """
+        logtype = config.get('type', PatroniLogger.DEFAULT_TYPE)
+        logformat: type_logformat = config.get('format', PatroniLogger.DEFAULT_FORMAT)
+        dateformat = config.get('dateformat') or None  # Convert empty string to `None`
+        static_fields = config.get('static_fields', {})
+
+        if dateformat is not None and not isinstance(dateformat, str):
+            _LOGGER.warning('Expected log dateformat to be a string, but got "%s"', type(dateformat))
+            dateformat = None
+
+        if logtype == 'json':
+            formatter = self._get_json_formatter(logformat, dateformat, static_fields)
+        else:
+            formatter = self._get_plain_formatter(logformat, dateformat)
+
+        return formatter
+
     def reload_config(self, config: Dict[str, Any]) -> None:
         """Apply log related configuration.
 
@@ -244,44 +407,41 @@ class PatroniLogger(Thread):
                 # show stack traces as ``ERROR`` log messages
                 logging.Logger.exception = error_exception
 
-            new_handler = None
+            handler = self.log_handler
+
             if 'dir' in config:
-                if not isinstance(self.log_handler, RotatingFileHandler):
+                if not isinstance(handler, RotatingFileHandler):
-                    new_handler = RotatingFileHandler(os.path.join(config['dir'], __name__))
+                    handler = RotatingFileHandler(os.path.join(config['dir'], __name__))
-                handler = new_handler or self.log_handler
+
-                if TYPE_CHECKING:  # pragma: no cover
-                    assert isinstance(handler, RotatingFileHandler)
                 handler.maxBytes = int(config.get('file_size', 25000000))  # pyright: ignore [reportGeneralTypeIssues]
                 handler.backupCount = int(config.get('file_num', 4))
-            else:
+            # we can't use `if not isinstance(handler, logging.StreamHandler)` below,
-                if self.log_handler is None or isinstance(self.log_handler, RotatingFileHandler):
+            # because RotatingFileHandler is a child of StreamHandler!!!
-                    new_handler = logging.StreamHandler()
+            elif handler is None or isinstance(handler, RotatingFileHandler):
-                handler = new_handler or self.log_handler
+                handler = logging.StreamHandler()
 
-            oldlogformat = (self._config or {}).get('format', PatroniLogger.DEFAULT_FORMAT)
+            is_new_handler = handler != self.log_handler
-            logformat = config.get('format', PatroniLogger.DEFAULT_FORMAT)
 
-            olddateformat = (self._config or {}).get('dateformat') or None
+            if (self._is_config_changed(config) or is_new_handler) and handler:
-            dateformat = config.get('dateformat') or None  # Convert empty string to `None`
+                formatter = self._get_formatter(config)
+                handler.setFormatter(formatter)
 
-            if (oldlogformat != logformat or olddateformat != dateformat or new_handler) and handler:
+            if is_new_handler:
-                handler.setFormatter(logging.Formatter(logformat, dateformat))
-
-            if new_handler:
                 with self.log_handler_lock:
                     if self.log_handler:
                         self._old_handlers.append(self.log_handler)
-                    self.log_handler = new_handler
+                    self.log_handler = handler
 
             self._config = config.copy()
-            self.update_loggers()
+            self.update_loggers(config.get('loggers') or {})
 
     def _close_old_handlers(self) -> None:
         """Close old log handlers.
 
         .. note::
             It is used to remove different handlers that were configured previous to a reload in the configuration,
-            e.g. if we are switching from :class:`RotatingFileHandler` to class:`StreamHandler` and vice-versa.
+            e.g. if we are switching from :class:`~logging.handlers.RotatingFileHandler` to
+            class:`~logging.StreamHandler` and vice-versa.
         """
         while True:
             with self.log_handler_lock:

patroni/postgresql/__init__.py

@@ -12,29 +12,29 @@ from datetime import datetime
 from dateutil import tz
 from psutil import TimeoutExpired
 from threading import current_thread, Lock
-from typing import Any, Callable, Dict, Generator, List, Optional, Union, Tuple, TYPE_CHECKING
+from typing import Any, Callable, Dict, Iterator, List, Optional, Union, Tuple, TYPE_CHECKING
 
 from .bootstrap import Bootstrap
 from .callback_executor import CallbackAction, CallbackExecutor
 from .cancellable import CancellableSubprocess
 from .config import ConfigHandler, mtime
-from .connection import Connection, get_connection_cursor
+from .connection import ConnectionPool, get_connection_cursor
-from .citus import CitusHandler
 from .misc import parse_history, parse_lsn, postgres_major_version_to_int
+from .mpp import AbstractMPP
 from .postmaster import PostmasterProcess
 from .slots import SlotsHandler
 from .sync import SyncHandler
-from .. import psycopg
+from .. import global_config, psycopg
 from ..async_executor import CriticalTask
-from ..collections import CaseInsensitiveSet
+from ..collections import CaseInsensitiveSet, CaseInsensitiveDict
-from ..dcs import Cluster, Leader, Member
+from ..dcs import Cluster, Leader, Member, SLOT_ADVANCE_AVAILABLE_VERSION
 from ..exceptions import PostgresConnectionException
 from ..utils import Retry, RetryFailedError, polling_loop, data_directory_is_empty, parse_int
+from ..tags import Tags
 
 if TYPE_CHECKING:  # pragma: no cover
     from psycopg import Connection as Connection3, Cursor
     from psycopg2 import connection as connection3, cursor
-    from ..config import GlobalConfig
 
 logger = logging.getLogger(__name__)
 
@@ -57,13 +57,13 @@ class Postgresql(object):
     TL_LSN = ("CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0 "
               "ELSE ('x' || pg_catalog.substr(pg_catalog.pg_{0}file_name("
               "pg_catalog.pg_current_{0}_{1}()), 1, 8))::bit(32)::int END, "  # primary timeline
-              "CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0 "
+              "CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0 ELSE "
-              "ELSE pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}_{1}(), '0/0')::bigint END, "  # write_lsn
+              "pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}{2}_{1}(), '0/0')::bigint END, "  # wal(_flush)?_lsn
               "pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), '0/0')::bigint, "
               "pg_catalog.pg_{0}_{1}_diff(COALESCE(pg_catalog.pg_last_{0}_receive_{1}(), '0/0'), '0/0')::bigint, "
               "pg_catalog.pg_is_in_recovery() AND pg_catalog.pg_is_{0}_replay_paused()")
 
-    def __init__(self, config: Dict[str, Any]) -> None:
+    def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None:
         self.name: str = config['name']
         self.scope: str = config['scope']
         self._data_dir: str = config['data_dir']
@@ -73,14 +73,14 @@ class Postgresql(object):
         self.connection_string: str
         self.proxy_url: Optional[str]
         self._major_version = self.get_major_version()
-        self._global_config = None
 
         self._state_lock = Lock()
         self.set_state('stopped')
 
-        self._pending_restart = False
+        self._pending_restart_reason = CaseInsensitiveDict()
-        self._connection = Connection()
+        self.connection_pool = ConnectionPool()
-        self.citus_handler = CitusHandler(self, config.get('citus'))
+        self._connection = self.connection_pool.get('heartbeat')
+        self.mpp_handler = mpp.get_handler_impl(self)
         self.config = ConfigHandler(self, config)
         self.config.check_directories()
 
@@ -111,24 +111,37 @@ class Postgresql(object):
         self._state_entry_timestamp = 0
 
         self._cluster_info_state = {}
-        self._has_permanent_logical_slots = True
+        self._has_permanent_slots = True
         self._enforce_hot_standby_feedback = False
         self._cached_replica_timeline = None
 
         # Last known running process
         self._postmaster_proc = None
 
-        if self.is_running():  # we are "joining" already running postgres
+        self._available_gucs = None
-            self.set_state('running')
+
-            self.set_role('master' if self.is_leader() else 'replica')
+        if self.is_running():
-            # postpone writing postgresql.conf for 12+ because recovery parameters are not yet known
+            # If we found postmaster process we need to figure out whether postgres is accepting connections
-            if self.major_version < 120000 or self.is_leader():
+            self.set_state('starting')
-                self.config.write_postgresql_conf()
+            self.check_startup_state_changed()
+
+        if self.state == 'running':  # we are "joining" already running postgres
+            # we know that PostgreSQL is accepting connections and can read some GUC's from pg_settings
+            self.config.load_current_server_parameters()
+
+            self.set_role('master' if self.is_primary() else 'replica')
+
             hba_saved = self.config.replace_pg_hba()
             ident_saved = self.config.replace_pg_ident()
-            if hba_saved or ident_saved:
+
+            if self.major_version < 120000 or self.role in ('master', 'primary'):
+                # If PostgreSQL is running as a primary or we run PostgreSQL that is older than 12 we can
+                # call reload_config() once again (the first call happened in the ConfigHandler constructor),
+                # so that it can figure out if config files should be updated and pg_ctl reload executed.
+                self.config.reload_config(config, sighup=bool(hba_saved or ident_saved))
+            elif hba_saved or ident_saved:
                 self.reload()
-        elif self.role in ('master', 'primary'):
+        elif not self.is_running() and self.role in ('master', 'primary'):
             self.set_role('demoted')
 
     @property
@@ -159,20 +172,36 @@ class Postgresql(object):
     def wal_name(self) -> str:
         return 'wal' if self._major_version >= 100000 else 'xlog'
 
+    @property
+    def wal_flush(self) -> str:
+        """For PostgreSQL 9.6 onwards we want to use pg_current_wal_flush_lsn()/pg_current_xlog_flush_location()."""
+        return '_flush' if self._major_version >= 90600 else ''
+
     @property
     def lsn_name(self) -> str:
         return 'lsn' if self._major_version >= 100000 else 'location'
 
+    @property
+    def supports_quorum_commit(self) -> bool:
+        """``True`` if quorum commit is supported by Postgres."""
+        return self._major_version >= 100000
+
     @property
     def supports_multiple_sync(self) -> bool:
         """:returns: `True` if Postgres version supports more than one synchronous node."""
         return self._major_version >= 90600
 
+    @property
+    def can_advance_slots(self) -> bool:
+        """``True`` if :attr:``major_version`` is greater than 110000."""
+        return self.major_version >= SLOT_ADVANCE_AVAILABLE_VERSION
+
     @property
     def cluster_info_query(self) -> str:
         """Returns the monitoring query with a fixed number of fields.
 
         The query text is constructed based on current state in DCS and PostgreSQL version:
+
         1. function names depend on version. wal/lsn for v10+ and xlog/location for pre v10.
         2. for primary we query timeline_id (extracted from pg_walfile_name()) and pg_current_wal_lsn()
         3. for replicas we query pg_last_wal_receive_lsn(), pg_last_wal_replay_lsn(), and  pg_is_wal_replay_paused()
@@ -182,7 +211,8 @@ class Postgresql(object):
         7. if sync replication is enabled we query pg_stat_replication and aggregate the result.
            In addition to that we get current values of synchronous_commit and synchronous_standby_names GUCs.
 
-        If some conditions are not satisfied we simply put static values instead. E.g., NULL, 0, '', and so on."""
+        If some conditions are not satisfied we simply put static values instead. E.g., NULL, 0, '', and so on.
+        """
 
         extra = ", " + (("pg_catalog.current_setting('synchronous_commit'), "
                          "pg_catalog.current_setting('synchronous_standby_names'), "
@@ -193,15 +223,16 @@ class Postgresql(object):
                          "FROM pg_catalog.pg_stat_get_wal_senders() w,"
                          " pg_catalog.pg_stat_get_activity(w.pid)"
                          " WHERE w.state = 'streaming') r)").format(self.wal_name, self.lsn_name)
-                        if (not self.global_config or self.global_config.is_synchronous_mode)
+                        if global_config.is_synchronous_mode
                         and self.role in ('master', 'primary', 'promoted') else "'on', '', NULL")
 
         if self._major_version >= 90600:
             extra = ("pg_catalog.current_setting('restore_command')" if self._major_version >= 120000 else "NULL") +\
                 ", " + ("(SELECT pg_catalog.json_agg(s.*) FROM (SELECT slot_name, slot_type as type, datoid::bigint, "
                         "plugin, catalog_xmin, pg_catalog.pg_wal_lsn_diff(confirmed_flush_lsn, '0/0')::bigint"
-                        " AS confirmed_flush_lsn FROM pg_catalog.pg_get_replication_slots()) AS s)"
+                        " AS confirmed_flush_lsn, pg_catalog.pg_wal_lsn_diff(restart_lsn, '0/0')::bigint"
-                        if self._has_permanent_logical_slots and self._major_version >= 110000 else "NULL") + extra
+                        " AS restart_lsn FROM pg_catalog.pg_get_replication_slots()) AS s)"
+                        if self._has_permanent_slots and self.can_advance_slots else "NULL") + extra
             extra = (", CASE WHEN latest_end_lsn IS NULL THEN NULL ELSE received_tli END,"
                      " slot_name, conninfo, status, {0} FROM pg_catalog.pg_stat_get_wal_receiver()").format(extra)
             if self.role == 'standby_leader':
@@ -211,12 +242,14 @@ class Postgresql(object):
         else:
             extra = "0, NULL, NULL, NULL, NULL, NULL, NULL" + extra
 
-        return ("SELECT " + self.TL_LSN + ", {2}").format(self.wal_name, self.lsn_name, extra)
+        return ("SELECT " + self.TL_LSN + ", {3}").format(self.wal_name, self.lsn_name, self.wal_flush, extra)
 
     @property
     def available_gucs(self) -> CaseInsensitiveSet:
         """GUCs available in this Postgres server."""
-        return self._get_gucs()
+        if not self._available_gucs:
+            self._available_gucs = self._get_gucs()
+        return self._available_gucs
 
     def _version_file_exists(self) -> bool:
         return not self.data_directory_empty() and os.path.isfile(self._version_file)
@@ -270,7 +303,7 @@ class Postgresql(object):
 
         :returns: 'ok' if PostgreSQL is up, 'reject' if starting up, 'no_resopnse' if not up."""
 
-        r = self.config.local_connect_kwargs
+        r = self.connection_pool.conn_kwargs
         cmd = [self.pgcommand('pg_isready'), '-p', r['port'], '-d', self._database]
 
         # Host is not set if we are connecting via default unix socket
@@ -293,11 +326,22 @@ class Postgresql(object):
         self._is_leader_retry.deadline = self.retry.deadline = config['retry_timeout'] / 2.0
 
     @property
-    def pending_restart(self) -> bool:
+    def pending_restart_reason(self) -> CaseInsensitiveDict:
-        return self._pending_restart
+        """Get :attr:`_pending_restart_reason` value.
 
-    def set_pending_restart(self, value: bool) -> None:
+        :attr:`_pending_restart_reason` is a :class:`CaseInsensitiveDict` object of the PG parameters that are
-        self._pending_restart = value
+        causing pending restart state. Every key is a parameter name, value - a dictionary containing the old
+        and the new value (see :func:`~patroni.postgresql.config.get_param_diff`).
+        """
+        return self._pending_restart_reason
+
+    def set_pending_restart_reason(self, diff_dict: CaseInsensitiveDict) -> None:
+        """Set new or update current :attr:`_pending_restart_reason`.
+
+        :param diff_dict: :class:``CaseInsensitiveDict`` object with the parameters that are causing pending restart
+            state with the diff of their values. Used to reset/update the :attr:`_pending_restart_reason`.
+        """
+        self._pending_restart_reason = diff_dict
 
     @property
     def sysid(self) -> str:
@@ -321,40 +365,50 @@ class Postgresql(object):
     def connection(self) -> Union['connection3', 'Connection3[Any]']:
         return self._connection.get()
 
-    def set_connection_kwargs(self, kwargs: Dict[str, Any]) -> None:
+    def _query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]:
-        self._connection.set_conn_kwargs(kwargs.copy())
+        """Execute *sql* query with *params* and optionally return results.
-        self.citus_handler.set_conn_kwargs(kwargs.copy())
 
-    def _query(self, sql: str, *params: Any) -> Union['Cursor[Any]', 'cursor']:
+        :param sql: SQL statement to execute.
-        """We are always using the same cursor, therefore this method is not thread-safe!!!
+        :param params: parameters to pass.
-        You can call it from different threads only if you are holding explicit `AsyncExecutor` lock,
+
-        because the main thread is always holding this lock when running HA cycle."""
+        :returns: a query response as a list of tuples if there is any.
-        cursor = None
+        :raises:
+            :exc:`~psycopg.Error` if had issues while executing *sql*.
+
+            :exc:`~patroni.exceptions.PostgresConnectionException`: if had issues while connecting to the database.
+
+            :exc:`~patroni.utils.RetryFailedError`: if it was detected that connection/query failed due to PostgreSQL
+            restart.
+        """
         try:
-            cursor = self._connection.cursor()
+            return self._connection.query(sql, *params)
-            cursor.execute(sql.encode('utf-8'), params or None)
+        except PostgresConnectionException as exc:
-            return cursor
-        except psycopg.Error as e:
-            if cursor and cursor.connection.closed == 0:
-                # When connected via unix socket, psycopg2 can't recoginze 'connection lost'
-                # and leaves `_cursor_holder.connection.closed == 0`, but psycopg2.OperationalError
-                # is still raised (what is correct). It doesn't make sense to continiue with existing
-                # connection and we will close it, to avoid its reuse by the `cursor` method.
-                if isinstance(e, psycopg.OperationalError):
-                    self._connection.close()
-                else:
-                    raise e
             if self.state == 'restarting':
-                raise RetryFailedError('cluster is being restarted')
+                raise RetryFailedError('cluster is being restarted') from exc
-            raise PostgresConnectionException('connection problems')
+            raise
 
-    def query(self, sql: str, *args: Any, **kwargs: Any) -> Union['Cursor[Any]', 'cursor']:
+    def query(self, sql: str, *params: Any, retry: bool = True) -> List[Tuple[Any, ...]]:
-        if not kwargs.get('retry', True):
+        """Execute *sql* query with *params* and optionally return results.
-            return self._query(sql, *args)
+
+        :param sql: SQL statement to execute.
+        :param params: parameters to pass.
+        :param retry: whether the query should be retried upon failure or given up immediately.
+
+        :returns: a query response as a list of tuples if there is any.
+        :raises:
+            :exc:`~psycopg.Error` if had issues while executing *sql*.
+
+            :exc:`~patroni.exceptions.PostgresConnectionException`: if had issues while connecting to the database.
+
+            :exc:`~patroni.utils.RetryFailedError`: if it was detected that connection/query failed due to PostgreSQL
+            restart or if retry deadline was exceeded.
+        """
+        if not retry:
+            return self._query(sql, *params)
         try:
-            return self.retry(self._query, sql, *args)
+            return self.retry(self._query, sql, *params)
-        except RetryFailedError as e:
+        except RetryFailedError as exc:
-            raise PostgresConnectionException(str(e))
+            raise PostgresConnectionException(str(exc)) from exc
 
     def pg_control_exists(self) -> bool:
         return os.path.isfile(self._pg_control)
@@ -391,46 +445,41 @@ class Postgresql(object):
                 self.config.write_postgresql_conf()
                 self.reload()
 
-    @property
+    def reset_cluster_info_state(self, cluster: Optional[Cluster], tags: Optional[Tags] = None) -> None:
-    def global_config(self) -> Optional['GlobalConfig']:
-        return self._global_config
-
-    def reset_cluster_info_state(self, cluster: Union[Cluster, None], nofailover: bool = False,
-                                 global_config: Optional['GlobalConfig'] = None) -> None:
         """Reset monitoring query cache.
 
-        It happens in the beginning of heart-beat loop and on change of `synchronous_standby_names`.
+        .. note::
+            It happens in the beginning of heart-beat loop and on change of `synchronous_standby_names`.
 
         :param cluster: currently known cluster state from DCS
-        :param nofailover: whether this node could become a new primary.
+        :param tags: reference to an object implementing :class:`Tags` interface.
-                           Important when there are logical permanent replication slots because "nofailover"
-                           node could do cascading replication and should enable `hot_standby_feedback`
-        :param global_config: last known :class:`GlobalConfig` object
         """
         self._cluster_info_state = {}
-        if cluster and cluster.config and cluster.config.modify_version:
-            self._has_permanent_logical_slots =\
-                cluster.has_permanent_logical_slots(self.name, nofailover, self.major_version)
 
+        if not tags:
+            return
+
+        if global_config.is_standby_cluster:
+            # Standby cluster can't have logical replication slots, and we don't need to enforce hot_standby_feedback
+            self.set_enforce_hot_standby_feedback(False)
+
+        if cluster and cluster.config and cluster.config.modify_version:
             # We want to enable hot_standby_feedback if the replica is supposed
             # to have a logical slot or in case if it is the cascading replica.
-            self.set_enforce_hot_standby_feedback(
+            self.set_enforce_hot_standby_feedback(not global_config.is_standby_cluster and self.can_advance_slots
-                self._has_permanent_logical_slots
+                                                  and cluster.should_enforce_hot_standby_feedback(self, tags))
-                or cluster.should_enforce_hot_standby_feedback(self.name, nofailover, self.major_version))
+            self._has_permanent_slots = cluster.has_permanent_slots(self, tags)
-
-        if global_config:
-            self._global_config = global_config
 
     def _cluster_info_state_get(self, name: str) -> Optional[Any]:
         if not self._cluster_info_state:
             try:
-                result = self._is_leader_retry(self._query, self.cluster_info_query).fetchone()
+                result = self._is_leader_retry(self._query, self.cluster_info_query)[0]
                 cluster_info_state = dict(zip(['timeline', 'wal_position', 'replayed_location',
                                                'received_location', 'replay_paused', 'pg_control_timeline',
                                                'received_tli', 'slot_name', 'conninfo', 'receiver_state',
                                                'restore_command', 'slots', 'synchronous_commit',
                                                'synchronous_standby_names', 'pg_stat_replication'], result))
-                if self._has_permanent_logical_slots:
+                if self._has_permanent_slots and self.can_advance_slots:
                     cluster_info_state['slots'] =\
                         self.slots_handler.process_permanent_slots(cluster_info_state['slots'])
                 self._cluster_info_state = cluster_info_state
@@ -474,14 +523,14 @@ class Postgresql(object):
         """:returns: a result set of 'SELECT * FROM pg_stat_replication'."""
         return self._cluster_info_state_get('pg_stat_replication') or []
 
-    def replication_state_from_parameters(self, is_leader: bool, receiver_state: Optional[str],
+    def replication_state_from_parameters(self, is_primary: bool, receiver_state: Optional[str],
                                           restore_command: Optional[str]) -> Optional[str]:
         """Figure out the replication state from input parameters.
 
         .. note::
             This method could be only called when Postgres is up, running and queries are successfuly executed.
 
-        :is_leader: `True` is postgres is not running in recovery
+        :is_primary: `True` is postgres is not running in recovery
         :receiver_state: value from `pg_stat_get_wal_receiver.state` or None if Postgres is older than 9.6
         :restore_command: value of ``restore_command`` GUC for PostgreSQL 12+ or
                           `postgresql.recovery_conf.restore_command` if it is set in Patroni configuration
@@ -490,7 +539,7 @@ class Postgresql(object):
                   - 'streaming' if replica is streaming according to the `pg_stat_wal_receiver` view;
                   - 'in archive recovery' if replica isn't streaming and there is a `restore_command`
         """
-        if self._major_version >= 90600 and not is_leader:
+        if self._major_version >= 90600 and not is_primary:
             if receiver_state == 'streaming':
                 return 'streaming'
             # For Postgres older than 12 we get `restore_command` from Patroni config, otherwise we check GUC
@@ -505,11 +554,11 @@ class Postgresql(object):
 
         :returns: ``streaming``, ``in archive recovery``, or ``None``
         """
-        return self.replication_state_from_parameters(self.is_leader(),
+        return self.replication_state_from_parameters(self.is_primary(),
                                                       self._cluster_info_state_get('receiver_state'),
                                                       self._cluster_info_state_get('restore_command'))
 
-    def is_leader(self) -> bool:
+    def is_primary(self) -> bool:
         try:
             return bool(self._cluster_info_state_get('timeline'))
         except PostgresConnectionException:
@@ -542,17 +591,20 @@ class Postgresql(object):
                              r'lsn: ([0-9A-Fa-f]+/[0-9A-Fa-f]+), prev ([0-9A-Fa-f]+/[0-9A-Fa-f]+), '
                              r'.*?desc: (.+)', out.decode('utf-8'))
             if match:
-                return match.groups()
+                return match.group(1), match.group(2), match.group(3), match.group(4)
         return None, None, None, None
 
-    def latest_checkpoint_location(self) -> Optional[int]:
+    def _checkpoint_locations_from_controldata(self, data: Dict[str, str]) -> Optional[Tuple[int, int]]:
-        """Returns checkpoint location for the cleanly shut down primary.
+        """Get shutdown checkpoint location.
-           But, if we know that the checkpoint was written to the new WAL
-           due to the archive_mode=on, we will return the LSN of prev wal record (SWITCH)."""
 
-        data = self.controldata()
+        :param data: :class:`dict` object with values returned by `pg_controldata` tool.
+
+        :returns: a tuple of checkpoint LSN for the cleanly shut down primary, and LSN of prev wal record (SWITCH)
+                  if we know that the checkpoint was written to the new WAL file due to the archive_mode=on.
+        """
         timeline = data.get("Latest checkpoint's TimeLineID")
         lsn = checkpoint_lsn = data.get('Latest checkpoint location')
+        prev_lsn = None
         if data.get('Database cluster state') == 'shut down' and lsn and timeline and checkpoint_lsn:
             try:
                 checkpoint_lsn = parse_lsn(checkpoint_lsn)
@@ -563,13 +615,26 @@ class Postgresql(object):
                     _, lsn, _, desc = self.parse_wal_record(timeline, prev)
                     prev = parse_lsn(prev)
                     # If the cluster is shutdown with archive_mode=on, WAL is switched before writing the checkpoint.
-                    # In this case we want to take the LSN of previous record (switch) as the last known WAL location.
+                    # In this case we want to take the LSN of previous record (SWITCH) as the last known WAL location.
                     if lsn and parse_lsn(lsn) == prev and str(desc).strip() in ('xlog switch', 'SWITCH'):
-                        return prev
+                        prev_lsn = prev
             except Exception as e:
                 logger.error('Exception when parsing WAL pg_%sdump output: %r', self.wal_name, e)
             if isinstance(checkpoint_lsn, int):
-                return checkpoint_lsn
+                return checkpoint_lsn, (prev_lsn or checkpoint_lsn)
+
+    def latest_checkpoint_location(self) -> Optional[int]:
+        """Get shutdown checkpoint location.
+
+        .. note::
+            In case if checkpoint was written to the new WAL file due to the archive_mode=on
+            we return LSN of the previous wal record (SWITCH).
+
+        :returns: checkpoint LSN for the cleanly shut down primary.
+        """
+        checkpoint_locations = self._checkpoint_locations_from_controldata(self.controldata())
+        if checkpoint_locations:
+            return checkpoint_locations[1]
 
     def is_running(self) -> Optional[PostmasterProcess]:
         """Returns PostmasterProcess if one is running on the data directory or None. If most recently seen process
@@ -665,7 +730,7 @@ class Postgresql(object):
         # the former node, otherwise, we might get a stalled one
         # after kill -9, which would report incorrect data to
         # patroni.
-        self._connection.close()
+        self.connection_pool.close()
 
         if self.is_running():
             logger.error('Cannot start PostgreSQL because one is already running.')
@@ -678,7 +743,7 @@ class Postgresql(object):
         self.set_role(role or self.get_postgres_role_from_data_directory())
 
         self.set_state('starting')
-        self._pending_restart = False
+        self.set_pending_restart_reason(CaseInsensitiveDict())
 
         try:
             if not self.ensure_major_version_is_known():
@@ -736,7 +801,7 @@ class Postgresql(object):
     def checkpoint(self, connect_kwargs: Optional[Dict[str, Any]] = None,
                    timeout: Optional[float] = None) -> Optional[str]:
         check_not_is_in_recovery = connect_kwargs is not None
-        connect_kwargs = connect_kwargs or self.config.local_connect_kwargs
+        connect_kwargs = connect_kwargs or self.connection_pool.conn_kwargs
         for p in ['connect_timeout', 'options']:
             connect_kwargs.pop(p, None)
         if timeout:
@@ -755,7 +820,7 @@ class Postgresql(object):
             return 'not accessible or not healty'
 
     def stop(self, mode: str = 'fast', block_callbacks: bool = False, checkpoint: Optional[bool] = None,
-             on_safepoint: Optional[Callable[..., Any]] = None, on_shutdown: Optional[Callable[[int], Any]] = None,
+             on_safepoint: Optional[Callable[..., Any]] = None, on_shutdown: Optional[Callable[[int, int], Any]] = None,
              before_shutdown: Optional[Callable[..., Any]] = None, stop_timeout: Optional[int] = None) -> bool:
         """Stop PostgreSQL
 
@@ -785,7 +850,7 @@ class Postgresql(object):
         return success
 
     def _do_stop(self, mode: str, block_callbacks: bool, checkpoint: bool,
-                 on_safepoint: Optional[Callable[..., Any]], on_shutdown: Optional[Callable[..., Any]],
+                 on_safepoint: Optional[Callable[..., Any]], on_shutdown: Optional[Callable[[int, int], Any]],
                  before_shutdown: Optional[Callable[..., Any]], stop_timeout: Optional[int]) -> Tuple[bool, bool]:
         postmaster = self.is_running()
         if not postmaster:
@@ -825,7 +890,9 @@ class Postgresql(object):
             while postmaster.is_running():
                 data = self.controldata()
                 if data.get('Database cluster state', '') == 'shut down':
-                    on_shutdown(self.latest_checkpoint_location())
+                    checkpoint_locations = self._checkpoint_locations_from_controldata(data)
+                    if checkpoint_locations:
+                        on_shutdown(*checkpoint_locations)
                     break
                 elif data.get('Database cluster state', '').startswith('shut down'):  # shut down in recovery
                     break
@@ -866,11 +933,10 @@ class Postgresql(object):
 
     def _wait_for_connection_close(self, postmaster: PostmasterProcess) -> None:
         try:
-            with self.connection().cursor() as cur:
+            while postmaster.is_running():  # Need a timeout here?
-                while postmaster.is_running():  # Need a timeout here?
+                self._connection.query("SELECT 1")
-                    cur.execute("SELECT 1")
+                time.sleep(STOP_POLLING_INTERVAL)
-                    time.sleep(STOP_POLLING_INTERVAL)
+        except (psycopg.Error, PostgresConnectionException):
-        except psycopg.Error:
             pass
 
     def reload(self, block_callbacks: bool = False) -> bool:
@@ -998,8 +1064,8 @@ class Postgresql(object):
             return None, None
 
     @contextmanager
-    def get_replication_connection_cursor(self, host: Optional[str] = None, port: int = 5432,
+    def get_replication_connection_cursor(self, host: Optional[str] = None, port: Union[int, str] = 5432,
-                                          **kwargs: Any) -> Generator[Union['cursor', 'Cursor[Any]'], None, None]:
+                                          **kwargs: Any) -> Iterator[Union['cursor', 'Cursor[Any]']]:
         conn_kwargs = self.config.replication.copy()
         conn_kwargs.update(host=host, port=int(port) if port else None, user=conn_kwargs.pop('username'),
                            connect_timeout=3, replication=1, options='-c statement_timeout=2000')
@@ -1125,8 +1191,8 @@ class Postgresql(object):
         except Exception as e:
             logger.error('Exception when calling `%s`: %r', cmd, e)
 
-    def promote(self, wait_seconds: int, task: CriticalTask, before_promote: Optional[Callable[..., Any]] = None,
+    def promote(self, wait_seconds: int, task: CriticalTask,
-                on_success: Optional[Callable[..., Any]] = None) -> Optional[bool]:
+                before_promote: Optional[Callable[..., Any]] = None) -> Optional[bool]:
         if self.role in ('promoted', 'master', 'primary'):
             return True
 
@@ -1147,21 +1213,19 @@ class Postgresql(object):
             before_promote()
 
         self.slots_handler.on_promote()
-        self.citus_handler.schedule_cache_rebuild()
+        self.mpp_handler.schedule_cache_rebuild()
 
         ret = self.pg_ctl('promote', '-W')
         if ret:
             self.set_role('promoted')
-            if on_success is not None:
-                on_success()
             self.call_nowait(CallbackAction.ON_ROLE_CHANGE)
             ret = self._wait_promote(wait_seconds)
         return ret
 
     @staticmethod
-    def _wal_position(is_leader: bool, wal_position: int,
+    def _wal_position(is_primary: bool, wal_position: int,
                       received_location: Optional[int], replayed_location: Optional[int]) -> int:
-        return wal_position if is_leader else max(received_location or 0, replayed_location or 0)
+        return wal_position if is_primary else max(received_location or 0, replayed_location or 0)
 
     def timeline_wal_position(self) -> Tuple[int, int, Optional[int]]:
         # This method could be called from different threads (simultaneously with some other `_query` calls).
@@ -1173,31 +1237,21 @@ class Postgresql(object):
             received_location = self.received_location()
             pg_control_timeline = self._cluster_info_state_get('pg_control_timeline')
         else:
-            with self.connection().cursor() as cursor:
+            timeline, wal_position, replayed_location, received_location, _, pg_control_timeline = \
-                cursor.execute(self.cluster_info_query.encode('utf-8'))
+                self._query(self.cluster_info_query)[0][:6]
-                row = cursor.fetchone()
-                if TYPE_CHECKING:  # pragma: no cover
-                    assert row is not None
-                (timeline, wal_position, replayed_location, received_location, _, pg_control_timeline) = row[:6]
 
         wal_position = self._wal_position(bool(timeline), wal_position, received_location, replayed_location)
-        return (timeline, wal_position, pg_control_timeline)
+        return timeline, wal_position, pg_control_timeline
 
     def postmaster_start_time(self) -> Optional[str]:
         try:
-            query = "SELECT " + self.POSTMASTER_START_TIME
+            sql = "SELECT " + self.POSTMASTER_START_TIME
-            if current_thread().ident == self.__thread_ident:
+            return self.query(sql, retry=current_thread().ident == self.__thread_ident)[0][0].isoformat(sep=' ')
-                row = self.query(query).fetchone()
-            else:
-                with self.connection().cursor() as cursor:
-                    cursor.execute(query)
-                    row = cursor.fetchone()
-            return row[0].isoformat(sep=' ') if row else None
         except psycopg.Error:
             return None
 
     def last_operation(self) -> int:
-        return self._wal_position(self.is_leader(), self._cluster_info_state_get('wal_position') or 0,
+        return self._wal_position(self.is_primary(), self._cluster_info_state_get('wal_position') or 0,
                                   self.received_location(), self.replayed_location())
 
     def configure_server_parameters(self) -> None:
@@ -1306,7 +1360,7 @@ class Postgresql(object):
         """
         self.ensure_major_version_is_known()
         self.slots_handler.schedule()
-        self.citus_handler.schedule_cache_rebuild()
+        self.mpp_handler.schedule_cache_rebuild()
         self._sysid = ''
 
     def _get_gucs(self) -> CaseInsensitiveSet:

patroni/postgresql/available_parameters/__init__.py

@@ -0,0 +1,58 @@
+import logging
+import sys
+
+from pathlib import Path
+from typing import Iterator
+
+logger = logging.getLogger(__name__)
+
+if sys.version_info < (3, 9):
+    PathLikeObj = Path
+    conf_dir = Path(__file__).parent
+else:
+    from importlib.resources import files
+
+    if sys.version_info < (3, 11):  # pragma: no cover
+        from importlib.abc import Traversable
+    else:  # pragma: no cover
+        from importlib.resources.abc import Traversable
+
+    PathLikeObj = Traversable
+    conf_dir = files(__name__)
+
+
+def get_validator_files() -> Iterator[PathLikeObj]:
+    """Recursively find YAML files from the current package directory.
+
+    :returns: an iterator of :class:`PathLikeObj` objects representing validator files.
+    """
+    return _traversable_walk(conf_dir.iterdir())
+
+
+def _traversable_walk(tvbs: Iterator[PathLikeObj]) -> Iterator[PathLikeObj]:
+    """Recursively walk through Path/Traversable objects, yielding all YAML files in deterministic order.
+
+    :param tvbs: An iterator over :class:`PathLikeObj` objects, where each object is a file or directory
+                 that potentially contains YAML files.
+
+    :yields: :class:`PathLikeObj` objects representing YAML files found during the traversal.
+    """
+    for tvb in _filter_and_sort_files(tvbs):
+        if tvb.is_file():
+            yield tvb
+        elif tvb.is_dir():
+            yield from _traversable_walk(tvb.iterdir())
+
+
+def _filter_and_sort_files(files: Iterator[PathLikeObj]) -> Iterator[PathLikeObj]:
+    """Sort files by name, and filter out non-YAML files and Python files.
+
+    :param files: A list of files and/or directories to be filtered and sorted.
+
+    :yields: filtered and sorted objects.
+    """
+    for file in sorted(files, key=lambda x: x.name):
+        if file.name.lower().endswith((".yml", ".yaml")) or file.is_dir():
+            yield file
+        elif not file.name.lower().endswith((".py", ".pyc")):
+            logger.info("Ignored a non-YAML file found under `%s` directory: `%s`.", __name__.split('.')[-1], file)

patroni/postgresql/bootstrap.py

@@ -100,10 +100,11 @@ class Bootstrap(object):
                     user_options.append('--{0}'.format(opt))
                 elif isinstance(opt, dict):
                     keys = list(opt.keys())
-                    if len(keys) != 1 or not isinstance(opt[keys[0]], str) or not option_is_allowed(keys[0]):
+                    if len(keys) == 1 and isinstance(opt[keys[0]], str) and option_is_allowed(keys[0]):
+                        user_options.append('--{0}={1}'.format(keys[0], unquote(opt[keys[0]])))
+                    else:
                         error_handler('Error when parsing {0} key-value option {1}: only one key-value is allowed'
                                       ' and value should be a string'.format(tool, opt[keys[0]]))
-                    user_options.append('--{0}={1}'.format(keys[0], unquote(opt[keys[0]])))
                 else:
                     error_handler('Error when parsing {0} option {1}: value should be string value'
                                   ' or a single key-value pair'.format(tool, opt))
@@ -151,9 +152,46 @@ class Bootstrap(object):
             os.unlink(trigger_file)
 
     def _custom_bootstrap(self, config: Any) -> bool:
+        """Bootstrap a fresh Patroni cluster using a custom method provided by the user.
+
+        :param config: configuration used for running a custom bootstrap method. It comes from the Patroni YAML file,
+            so it is expected to be a :class:`dict`.
+
+        .. note::
+            *config* must contain a ``command`` key, which value is the command or script to perform the custom
+            bootstrap procedure. The exit code of the ``command`` dictates if the bootstrap succeeded or failed.
+
+            When calling ``command``, Patroni will pass the following arguments to the ``command`` call:
+
+                * ``--scope``: contains the value of ``scope`` configuration;
+                * ``--data_dir``: contains the value of the ``postgresql.data_dir`` configuration.
+
+            You can avoid that behavior by filling the optional key ``no_params`` with the value ``False`` in the
+            configuration file, which will instruct Patroni to not pass these parameters to the ``command`` call.
+
+            Besides that, a couple more keys are supported in *config*, but optional:
+
+                * ``keep_existing_recovery_conf``: if ``True``, instruct Patroni to not remove the existing
+                  ``recovery.conf`` (PostgreSQL <= 11), to not discard recovery parameters from the configuration
+                  (PostgreSQL >= 12), and to not remove the files ``recovery.signal`` or ``standby.signal``
+                  (PostgreSQL >= 12). This is specially useful when you are restoring backups through tools like
+                  pgBackRest and Barman, in which case they generated the appropriate recovery settings for you;
+                * ``recovery_conf``: a section containing a map, where each key is the name of a recovery related
+                  setting, and the value is the value of the corresponding setting.
+
+            Any key/value other than the ones that were described above will be interpreted as additional arguments for
+            the ``command`` call. They will all be added to the call in the format ``--key=value``.
+
+        :returns: ``True`` if the bootstrap was successful, i.e. the execution of the custom ``command`` from *config*
+            exited with code ``0``, ``False`` otherwise.
+        """
         self._postgresql.set_state('running custom bootstrap script')
         params = [] if config.get('no_params') else ['--scope=' + self._postgresql.scope,
                                                      '--datadir=' + self._postgresql.data_dir]
+        # Add custom parameters specified by the user
+        reserved_args = {'command', 'no_params', 'keep_existing_recovery_conf', 'recovery_conf', 'scope', 'datadir'}
+        params += [f"--{arg}={val}" for arg, val in config.items() if arg not in reserved_args]
+
         try:
             logger.info('Running custom bootstrap script: %s', config['command'])
             if self._postgresql.cancellable.call(shlex.split(config['command']) + params) != 0:
@@ -176,7 +214,7 @@ class Bootstrap(object):
         """
         cmd = config.get('post_bootstrap') or config.get('post_init')
         if cmd:
-            r = self._postgresql.config.local_connect_kwargs
+            r = self._postgresql.connection_pool.conn_kwargs
             connstring = self._postgresql.config.format_dsn(r, True)
             if 'host' not in r:
                 # https://www.postgresql.org/docs/current/static/libpq-pgpass.html
@@ -400,6 +438,9 @@ BEGIN
 END;$$""".format(f, quote_ident(rewind['username'], postgresql.connection()))
                         postgresql.query(sql)
 
+                if config.get('users'):
+                    logger.warning('User creation via "bootstrap.users" will be removed in v4.0.0')
+
                 for name, value in (config.get('users') or {}).items():
                     if all(name != a.get('username') for a in (superuser, replication, rewind)):
                         self.create_or_update_role(name, value.get('password'), value.get('options', []))
@@ -423,15 +464,15 @@ END;$$""".format(f, quote_ident(rewind['username'], postgresql.connection()))
                         postgresql.restart()
                     else:
                         postgresql.config.replace_pg_hba()
-                        if postgresql.pending_restart:
+                        if postgresql.pending_restart_reason:
                             postgresql.restart()
                         else:
                             postgresql.reload()
                             time.sleep(1)  # give a time to postgres to "reload" configuration files
                             postgresql.connection().close()  # close connection to reconnect with a new password
                 else:  # initdb
-                    # We may want create database and extension for citus
+                    # We may want create database and extension for some MPP clusters
-                    self._postgresql.citus_handler.bootstrap()
+                    self._postgresql.mpp_handler.bootstrap()
         except Exception:
             logger.exception('post_bootstrap')
             task.complete(False)

patroni/postgresql/callback_executor.py

@@ -1,8 +1,9 @@
 import logging
+import sys
 
 from enum import Enum
 from threading import Condition, Thread
-from typing import List
+from typing import Any, Dict, List
 
 from .cancellable import CancellableExecutor, CancellableSubprocess
 
@@ -30,7 +31,9 @@ class OnReloadExecutor(CancellableSubprocess):
         self.cancel(kill=True)
         self._kill_children()
         with self._lock:
-            self._start_process(cmd, close_fds=True)
+            started = self._start_process(cmd, close_fds=True)
+        if started and self._process is not None:
+            Thread(target=self._process.wait).start()
 
 
 class CallbackExecutor(CancellableExecutor, Thread):
@@ -51,6 +54,8 @@ class CallbackExecutor(CancellableExecutor, Thread):
         If it couldn't be killed we wait until it finishes.
 
         :param cmd: command to be executed"""
+        kwargs: Dict[str, Any] = {'stacklevel': 3} if sys.version_info >= (3, 8) else {}
+        logger.debug('CallbackExecutor.call(%s)', cmd, **kwargs)
 
         if cmd[-3] == CallbackAction.ON_RELOAD:
             return self._on_reload_executor.call_nowait(cmd)

patroni/postgresql/config.py

@@ -6,16 +6,20 @@ import socket
 import stat
 import time
 
+from contextlib import contextmanager
 from urllib.parse import urlparse, parse_qsl, unquote
 from types import TracebackType
-from typing import Any, Collection, Dict, List, Optional, Union, Tuple, Type, TYPE_CHECKING
+from typing import Any, Callable, Collection, Dict, Iterator, List, Optional, Union, Tuple, Type, TYPE_CHECKING
 
 from .validator import recovery_parameters, transform_postgresql_parameter_value, transform_recovery_parameter_value
+from .. import global_config
 from ..collections import CaseInsensitiveDict, CaseInsensitiveSet
 from ..dcs import Leader, Member, RemoteMember, slot_name_from_member_name
-from ..exceptions import PatroniFatalException
+from ..exceptions import PatroniFatalException, PostgresConnectionException
-from ..utils import compare_values, parse_bool, parse_int, split_host_port, uri, validate_directory, is_subpath
+from ..file_perm import pg_perm
-from ..validator import IntValidator
+from ..utils import (compare_values, maybe_convert_from_base_unit, parse_bool, parse_int,
+                     split_host_port, uri, validate_directory, is_subpath)
+from ..validator import IntValidator, EnumValidator
 
 if TYPE_CHECKING:  # pragma: no cover
     from . import Postgresql
@@ -242,9 +246,10 @@ class ConfigWriter(object):
             self._fd.write(line)
             self._fd.write('\n')
 
-    def writelines(self, lines: List[str]) -> None:
+    def writelines(self, lines: List[Optional[str]]) -> None:
         for line in lines:
-            self.writeline(line)
+            if isinstance(line, str):
+                self.writeline(line)
 
     @staticmethod
     def escape(value: Any) -> str:  # Escape (by doubling) any single quotes or backslashes in given string
@@ -258,14 +263,37 @@ def _false_validator(value: Any) -> bool:
     return False
 
 
-def _wal_level_validator(value: Any) -> bool:
-    return str(value).lower() in ('hot_standby', 'replica', 'logical')
-
-
 def _bool_validator(value: Any) -> bool:
     return parse_bool(value) is not None
 
 
+def _bool_is_true_validator(value: Any) -> bool:
+    return parse_bool(value) is True
+
+
+def get_param_diff(old_value: Any, new_value: Any,
+                   vartype: Optional[str] = None, unit: Optional[str] = None) -> Dict[str, str]:
+    """Get a dictionary representing a single PG parameter's value diff.
+
+    :param old_value: current :class:`str` parameter value.
+    :param new_value: :class:`str` value of the paramater after a restart.
+    :param vartype: the target type to parse old/new_value. See ``vartype`` argument of
+        :func:`~patroni.utils.maybe_convert_from_base_unit`.
+    :param unit: unit of *old/new_value*. See ``base_unit`` argument of
+        :func:`~patroni.utils.maybe_convert_from_base_unit`.
+
+    :returns: a :class:`dict` object that contains two keys: ``old_value`` and ``new_value``
+        with their values casted to :class:`str` and converted from base units (if possible).
+    """
+    str_value: Callable[[Any], str] = lambda x: '' if x is None else str(x)
+    return {
+        'old_value': (maybe_convert_from_base_unit(str_value(old_value), vartype, unit)
+                      if vartype else str_value(old_value)),
+        'new_value': (maybe_convert_from_base_unit(str_value(new_value), vartype, unit)
+                      if vartype else str_value(new_value))
+    }
+
+
 class ConfigHandler(object):
 
     # List of parameters which must be always passed to postmaster as command line options
@@ -286,8 +314,8 @@ class ConfigHandler(object):
         'listen_addresses': (None, _false_validator, 90100),
         'port': (None, _false_validator, 90100),
         'cluster_name': (None, _false_validator, 90500),
-        'wal_level': ('hot_standby', _wal_level_validator, 90100),
+        'wal_level': ('hot_standby', EnumValidator(('hot_standby', 'replica', 'logical')), 90100),
-        'hot_standby': ('on', _false_validator, 90100),
+        'hot_standby': ('on', _bool_is_true_validator, 90100),
         'max_connections': (100, IntValidator(min=25), 90100),
         'max_wal_senders': (10, IntValidator(min=3), 90100),
         'wal_keep_segments': (8, IntValidator(min=1), 90100),
@@ -297,7 +325,7 @@ class ConfigHandler(object):
         'track_commit_timestamp': ('off', _bool_validator, 90500),
         'max_replication_slots': (10, IntValidator(min=4), 90400),
         'max_worker_processes': (8, IntValidator(min=2), 90400),
-        'wal_log_hints': ('on', _false_validator, 90400)
+        'wal_log_hints': ('on', _bool_is_true_validator, 90400)
     })
 
     _RECOVERY_PARAMETERS = CaseInsensitiveSet(recovery_parameters.keys())
@@ -324,14 +352,34 @@ class ConfigHandler(object):
                                         .format(self._pgpass))
         self._passfile = None
         self._passfile_mtime = None
-        self._synchronous_standby_names = None
         self._postmaster_ctime = None
         self._current_recovery_params: Optional[CaseInsensitiveDict] = None
         self._config = {}
         self._recovery_params = CaseInsensitiveDict()
-        self._server_parameters: CaseInsensitiveDict
+        self._server_parameters: CaseInsensitiveDict = CaseInsensitiveDict()
         self.reload_config(config)
 
+    def load_current_server_parameters(self) -> None:
+        """Read GUC's values from ``pg_settings`` when Patroni is joining the the postgres that is already running."""
+        exclude = [name.lower() for name, value in self.CMDLINE_OPTIONS.items() if value[1] == _false_validator]
+        keep_values = {k: self._server_parameters[k] for k in exclude}
+        server_parameters = CaseInsensitiveDict({r[0]: r[1] for r in self._postgresql.query(
+            "SELECT name, pg_catalog.current_setting(name) FROM pg_catalog.pg_settings"
+            " WHERE (source IN ('command line', 'environment variable') OR sourcefile = %s)"
+            " AND pg_catalog.lower(name) != ALL(%s)", self._postgresql_conf, exclude)})
+        recovery_params = CaseInsensitiveDict({k: server_parameters.pop(k) for k in self._RECOVERY_PARAMETERS
+                                               if k in server_parameters})
+        # We also want to load current settings of recovery parameters, including primary_conninfo
+        # and primary_slot_name, otherwise patronictl restart will update postgresql.conf
+        # and remove them, what in the worst case will cause another restart.
+        # We are doing it only for PostgresSQL v12 onwards, because older version still have recovery.conf
+        if not self._postgresql.is_primary() and self._postgresql.major_version >= 120000:
+            # primary_conninfo is expected to be a dict, therefore we need to parse it
+            recovery_params['primary_conninfo'] = parse_dsn(recovery_params.pop('primary_conninfo', '')) or {}
+            self._recovery_params = recovery_params
+
+        self._server_parameters = CaseInsensitiveDict({**server_parameters, **keep_values})
+
     def setup_server_parameters(self) -> None:
         self._server_parameters = self.get_server_parameters(self._config)
         self._adjust_recovery_parameters()
@@ -367,6 +415,30 @@ class ConfigHandler(object):
             configuration.append('pg_ident.conf')
         return configuration
 
+    def set_file_permissions(self, filename: str) -> None:
+        """Set permissions of file *filename* according to the expected permissions if it resides under PGDATA.
+
+        .. note::
+            Do nothing if the file is not under PGDATA.
+
+        :param filename: path to a file which permissions might need to be adjusted.
+        """
+        if is_subpath(self._postgresql.data_dir, filename):
+            pg_perm.set_permissions_from_data_directory(self._postgresql.data_dir)
+            os.chmod(filename, pg_perm.file_create_mode)
+
+    @contextmanager
+    def config_writer(self, filename: str) -> Iterator[ConfigWriter]:
+        """Create :class:`ConfigWriter` object and set permissions on a *filename*.
+
+        :param filename: path to a config file.
+
+        :yields: :class:`ConfigWriter` object.
+        """
+        with ConfigWriter(filename) as writer:
+            yield writer
+        self.set_file_permissions(filename)
+
     def save_configuration_files(self, check_custom_bootstrap: bool = False) -> bool:
         """
             copy postgresql.conf to postgresql.conf.backup to be able to retrieve configuration files
@@ -380,6 +452,7 @@ class ConfigHandler(object):
                     backup_file = os.path.join(self._postgresql.data_dir, f + '.backup')
                     if os.path.isfile(config_file):
                         shutil.copy(config_file, backup_file)
+                        self.set_file_permissions(backup_file)
             except IOError:
                 logger.exception('unable to create backup copies of configuration files')
         return True
@@ -393,9 +466,11 @@ class ConfigHandler(object):
                 if not os.path.isfile(config_file):
                     if os.path.isfile(backup_file):
                         shutil.copy(backup_file, config_file)
+                        self.set_file_permissions(config_file)
                     # Previously we didn't backup pg_ident.conf, if file is missing just create empty
                     elif f == 'pg_ident.conf':
                         open(config_file, 'w').close()
+                        self.set_file_permissions(config_file)
         except IOError:
             logger.exception('unable to restore configuration files from backup')
 
@@ -409,7 +484,7 @@ class ConfigHandler(object):
         if self._postgresql.enforce_hot_standby_feedback:
             configuration['hot_standby_feedback'] = 'on'
 
-        with ConfigWriter(self._postgresql_conf) as f:
+        with self.config_writer(self._postgresql_conf) as f:
             include = self._config.get('custom_conf') or self._postgresql_base_conf_name
             f.writeline("include '{0}'\n".format(ConfigWriter.escape(include)))
             for name, value in sorted((configuration).items()):
@@ -439,6 +514,7 @@ class ConfigHandler(object):
         if not self.hba_file and not self._config.get('pg_hba'):
             with open(self._pg_hba_conf, 'a') as f:
                 f.write('\n{}\n'.format('\n'.join(config)))
+            self.set_file_permissions(self._pg_hba_conf)
         return True
 
     def replace_pg_hba(self) -> Optional[bool]:
@@ -458,14 +534,14 @@ class ConfigHandler(object):
                                   self.local_replication_address['host'], self.local_replication_address['port'],
                                   0, socket.SOCK_STREAM, socket.IPPROTO_TCP)})
 
-            with ConfigWriter(self._pg_hba_conf) as f:
+            with self.config_writer(self._pg_hba_conf) as f:
                 for address, t in addresses.items():
                     f.writeline((
                         '{0}\treplication\t{1}\t{3}\ttrust\n'
                         '{0}\tall\t{2}\t{3}\ttrust'
                     ).format(t, self.replication['username'], self._superuser.get('username') or 'all', address))
         elif not self.hba_file and self._config.get('pg_hba'):
-            with ConfigWriter(self._pg_hba_conf) as f:
+            with self.config_writer(self._pg_hba_conf) as f:
                 f.writelines(self._config['pg_hba'])
             return True
 
@@ -478,7 +554,7 @@ class ConfigHandler(object):
         """
 
         if not self.ident_file and self._config.get('pg_ident'):
-            with ConfigWriter(self._pg_ident_conf) as f:
+            with self.config_writer(self._pg_ident_conf) as f:
                 f.writelines(self._config['pg_ident'])
             return True
 
@@ -556,15 +632,14 @@ class ConfigHandler(object):
         is_remote_member = isinstance(member, RemoteMember)
         primary_conninfo = self.primary_conninfo_params(member)
         if primary_conninfo:
-            use_slots = self.get('use_slots', True) and self._postgresql.major_version >= 90400
+            use_slots = global_config.use_slots and self._postgresql.major_version >= 90400
             if use_slots and not (is_remote_member and member.no_replication_slot):
                 primary_slot_name = member.primary_slot_name if is_remote_member else self._postgresql.name
                 recovery_params['primary_slot_name'] = slot_name_from_member_name(primary_slot_name)
                 # We are a standby leader and are using a replication slot. Make sure we connect to
                 # the leader of the main cluster (in case more than one host is specified in the
                 # connstr) by adding 'target_session_attrs=read-write' to primary_conninfo.
-                if is_remote_member and 'target_sesions_attrs' not in primary_conninfo and\
+                if is_remote_member and ',' in primary_conninfo['host'] and self._postgresql.major_version >= 100000:
-                        self._postgresql.major_version >= 100000:
                     primary_conninfo['target_session_attrs'] = 'read-write'
             recovery_params['primary_conninfo'] = primary_conninfo
 
@@ -593,7 +668,24 @@ class ConfigHandler(object):
                                           'recovery_target_action', 'standby_mode', self._triggerfile_wrong_name})
         return CaseInsensitiveSet(self._RECOVERY_PARAMETERS - skip_params)
 
-    def _read_recovery_params(self) -> Tuple[Optional[CaseInsensitiveDict], Optional[bool]]:
+    def _read_recovery_params(self) -> Tuple[Optional[CaseInsensitiveDict], bool]:
+        """Read current recovery parameters values.
+
+        .. note::
+            We query Postgres only if we detected that Postgresql was restarted
+            or when at least one of the following files was updated:
+
+                * ``postgresql.conf``;
+                * ``postgresql.auto.conf``;
+                * ``passfile`` that is used in the ``primary_conninfo``.
+
+        :returns: a tuple with two elements:
+
+            * :class:`CaseInsensitiveDict` object with current values of recovery parameters,
+              or ``None`` if no configuration files were updated;
+
+            * ``True`` if new values of recovery parameters were queried, ``False`` otherwise.
+        """
         if self._postgresql.is_starting():
             return None, False
 
@@ -614,11 +706,20 @@ class ConfigHandler(object):
             self._postgresql_conf_mtime = pg_conf_mtime
             self._auto_conf_mtime = auto_conf_mtime
             self._postmaster_ctime = postmaster_ctime
-        except Exception:
+        except Exception as exc:
+            if all((isinstance(exc, PostgresConnectionException),
+                    self._postgresql_conf_mtime == pg_conf_mtime,
+                    self._auto_conf_mtime == auto_conf_mtime,
+                    self._passfile_mtime == passfile_mtime,
+                    self._postmaster_ctime != postmaster_ctime)):
+                # We detected that the connection to postgres fails, but the process creation time of the postmaster
+                # doesn't match the old value. It is an indicator that Postgres crashed and either doing crash
+                # recovery or down. In this case we return values like nothing changed in the config.
+                return None, False
             values = None
         return values, True
 
-    def _read_recovery_params_pre_v12(self) -> Tuple[Optional[CaseInsensitiveDict], Optional[bool]]:
+    def _read_recovery_params_pre_v12(self) -> Tuple[Optional[CaseInsensitiveDict], bool]:
         recovery_conf_mtime = mtime(self._recovery_conf)
         passfile_mtime = mtime(self._passfile) if self._passfile else False
         if recovery_conf_mtime == self._recovery_conf_mtime and passfile_mtime == self._passfile_mtime:
@@ -800,9 +901,11 @@ class ConfigHandler(object):
         if self._postgresql.major_version >= 120000:
             if parse_bool(recovery_params.pop('standby_mode', None)):
                 open(self._standby_signal, 'w').close()
+                self.set_file_permissions(self._standby_signal)
             else:
                 self._remove_file_if_exists(self._standby_signal)
                 open(self._recovery_signal, 'w').close()
+                self.set_file_permissions(self._recovery_signal)
 
             def restart_required(name: str) -> bool:
                 if self._postgresql.major_version >= 140000:
@@ -813,8 +916,7 @@ class ConfigHandler(object):
             self._current_recovery_params = CaseInsensitiveDict({n: [v, restart_required(n), self._postgresql_conf]
                                                                  for n, v in recovery_params.items()})
         else:
-            with ConfigWriter(self._recovery_conf) as f:
+            with self.config_writer(self._recovery_conf) as f:
-                os.chmod(self._recovery_conf, stat.S_IWRITE | stat.S_IREAD)
                 self._write_recovery_params(f, recovery_params)
 
     def remove_recovery_conf(self) -> None:
@@ -843,6 +945,7 @@ class ConfigHandler(object):
         if overwrite:
             try:
                 with open(self._auto_conf, 'w') as f:
+                    self.set_file_permissions(self._auto_conf)
                     for raw_line in lines:
                         f.write(raw_line)
             except Exception:
@@ -863,15 +966,16 @@ class ConfigHandler(object):
         parameters = config['parameters'].copy()
         listen_addresses, port = split_host_port(config['listen'], 5432)
         parameters.update(cluster_name=self._postgresql.scope, listen_addresses=listen_addresses, port=str(port))
-        if not self._postgresql.global_config or self._postgresql.global_config.is_synchronous_mode:
+        if global_config.is_synchronous_mode:
-            if self._synchronous_standby_names is None:
+            synchronous_standby_names = self._server_parameters.get('synchronous_standby_names')
-                if self._postgresql.global_config and self._postgresql.global_config.is_synchronous_mode_strict\
+            if synchronous_standby_names is None:
+                if global_config.is_synchronous_mode_strict\
                         and self._postgresql.role in ('master', 'primary', 'promoted'):
                     parameters['synchronous_standby_names'] = '*'
                 else:
                     parameters.pop('synchronous_standby_names', None)
             else:
-                parameters['synchronous_standby_names'] = self._synchronous_standby_names
+                parameters['synchronous_standby_names'] = synchronous_standby_names
 
         # Handle hot_standby <-> replica rename
         if parameters.get('wal_level') == ('hot_standby' if self._postgresql.major_version >= 90600 else 'replica'):
@@ -887,7 +991,7 @@ class ConfigHandler(object):
             wal_keep_size = parse_int(parameters.pop('wal_keep_size', self.CMDLINE_OPTIONS['wal_keep_size'][0]), 'MB')
             parameters.setdefault('wal_keep_segments', int(((wal_keep_size or 0) + 8) / 16))
 
-        self._postgresql.citus_handler.adjust_postgres_gucs(parameters)
+        self._postgresql.mpp_handler.adjust_postgres_gucs(parameters)
 
         ret = CaseInsensitiveDict({k: v for k, v in parameters.items() if not self._postgresql.major_version
                                    or self._postgresql.major_version >= self.CMDLINE_OPTIONS.get(k, (0, 1, 90100))[2]})
@@ -910,24 +1014,32 @@ class ConfigHandler(object):
                 return 'localhost'  # connection via localhost is preferred
         return listen_addresses[0].strip()  # can't use localhost, take first address from listen_addresses
 
-    @property
-    def local_connect_kwargs(self) -> Dict[str, Any]:
-        ret = self._local_address.copy()
-        # add all of the other connection settings that are available
-        ret.update(self._superuser)
-        # if the "username" parameter is present, it actually needs to be "user"
-        # for connecting to PostgreSQL
-        if 'username' in self._superuser:
-            ret['user'] = self._superuser['username']
-            del ret['username']
-        # ensure certain Patroni configurations are available
-        ret.update({'dbname': self._postgresql.database,
-                    'fallback_application_name': 'Patroni',
-                    'connect_timeout': 3,
-                    'options': '-c statement_timeout=2000'})
-        return ret
-
     def resolve_connection_addresses(self) -> None:
+        """Calculates and sets local and remote connection urls and options.
+
+        This method sets:
+            * :attr:`Postgresql.connection_string <patroni.postgresql.Postgresql.connection_string>` attribute, which
+              is later written to the member key in DCS as ``conn_url``.
+            * :attr:`ConfigHandler.local_replication_address` attribute, which is used for replication connections to
+              local postgres.
+            * :attr:`ConnectionPool.conn_kwargs <patroni.postgresql.connection.ConnectionPool.conn_kwargs>` attribute,
+              which is used for superuser connections to local postgres.
+
+        .. note::
+            If there is a valid directory in ``postgresql.parameters.unix_socket_directories`` in the Patroni
+            configuration and ``postgresql.use_unix_socket`` and/or ``postgresql.use_unix_socket_repl``
+            are set to ``True``, we respectively use unix sockets for superuser and replication connections
+            to local postgres.
+
+            If there is a requirement to use unix sockets, but nothing is set in the
+            ``postgresql.parameters.unix_socket_directories``, we omit a ``host`` in connection parameters relying
+            on the ability of ``libpq`` to connect via some default unix socket directory.
+
+            If unix sockets are not requested we "switch" to TCP, prefering to use ``localhost`` if it is possible
+            to deduce that Postgres is listening on a local interface address.
+
+            Otherwise we just used the first address specified in the ``listen_addresses`` GUC.
+        """
         port = self._server_parameters['port']
         tcp_local_address = self._get_tcp_local_address()
         netloc = self._config.get('connect_address') or tcp_local_address + ':' + port
@@ -940,24 +1052,34 @@ class ConfigHandler(object):
 
         tcp_local_address = {'host': tcp_local_address, 'port': port}
 
-        self._local_address = unix_local_address if self._config.get('use_unix_socket') else tcp_local_address
         self.local_replication_address = unix_local_address\
             if self._config.get('use_unix_socket_repl') else tcp_local_address
 
         self._postgresql.connection_string = uri('postgres', netloc, self._postgresql.database)
-        self._postgresql.set_connection_kwargs(self.local_connect_kwargs)
 
-    def _get_pg_settings(
+        local_address = unix_local_address if self._config.get('use_unix_socket') else tcp_local_address
-            self, names: Collection[str]
+        local_conn_kwargs = {
-    ) -> Dict[str, Tuple[str, str, Optional[str], str, str, Optional[str]]]:
+            **local_address,
+            **self._superuser,
+            'dbname': self._postgresql.database,
+            'fallback_application_name': 'Patroni',
+            'connect_timeout': 3,
+            'options': '-c statement_timeout=2000'
+        }
+        # if the "username" parameter is present, it actually needs to be "user" for connecting to PostgreSQL
+        if 'username' in local_conn_kwargs:
+            local_conn_kwargs['user'] = local_conn_kwargs.pop('username')
+        # "notify" connection_pool about the "new" local connection address
+        self._postgresql.connection_pool.conn_kwargs = local_conn_kwargs
+
+    def _get_pg_settings(self, names: Collection[str]) -> Dict[Any, Tuple[Any, ...]]:
         return {r[0]: r for r in self._postgresql.query(('SELECT name, setting, unit, vartype, context, sourcefile'
                                                          + ' FROM pg_catalog.pg_settings '
                                                          + ' WHERE pg_catalog.lower(name) = ANY(%s)'),
                                                         [n.lower() for n in names])}
 
     @staticmethod
-    def _handle_wal_buffers(old_values: Dict[str, Tuple[str, str, Optional[str], str, str, Optional[str]]],
+    def _handle_wal_buffers(old_values: Dict[Any, Tuple[Any, ...]], changes: CaseInsensitiveDict) -> None:
-                            changes: CaseInsensitiveDict) -> None:
         wal_block_size = parse_int(old_values['wal_block_size'][1]) or 8192
         wal_segment_size = old_values['wal_segment_size']
         wal_segment_unit = parse_int(wal_segment_size[2], 'B') or 8192 \
@@ -978,13 +1100,15 @@ class ConfigHandler(object):
     def reload_config(self, config: Dict[str, Any], sighup: bool = False) -> None:
         self._superuser = config['authentication'].get('superuser', {})
         server_parameters = self.get_server_parameters(config)
+        params_skip_changes = CaseInsensitiveSet((*self._RECOVERY_PARAMETERS, 'hot_standby', 'wal_log_hints'))
 
-        conf_changed = hba_changed = ident_changed = local_connection_address_changed = pending_restart = False
+        conf_changed = hba_changed = ident_changed = local_connection_address_changed = False
+        param_diff = CaseInsensitiveDict()
         if self._postgresql.state == 'running':
             changes = CaseInsensitiveDict({p: v for p, v in server_parameters.items()
-                                           if p.lower() not in self._RECOVERY_PARAMETERS})
+                                           if p not in params_skip_changes})
             changes.update({p: None for p in self._server_parameters.keys()
-                            if not (p in changes or p.lower() in self._RECOVERY_PARAMETERS)})
+                            if not (p in changes or p in params_skip_changes)})
             if changes:
                 undef = []
                 if 'wal_buffers' in changes:  # we need to calculate the default value of wal_buffers
@@ -1003,20 +1127,28 @@ class ConfigHandler(object):
                         if new_value is None or not compare_values(r[3], r[2], r[1], new_value):
                             conf_changed = True
                             if r[4] == 'postmaster':
-                                pending_restart = True
+                                param_diff[r[0]] = get_param_diff(r[1], new_value, r[3], r[2])
-                                logger.info('Changed %s from %s to %s (restart might be required)',
+                                logger.info("Changed %s from '%s' to '%s' (restart might be required)",
-                                            r[0], r[1], new_value)
+                                            r[0], param_diff[r[0]]['old_value'], new_value)
                                 if config.get('use_unix_socket') and r[0] == 'unix_socket_directories'\
                                         or r[0] in ('listen_addresses', 'port'):
                                     local_connection_address_changed = True
                             else:
-                                logger.info('Changed %s from %s to %s', r[0], r[1], new_value)
+                                logger.info("Changed %s from '%s' to '%s'",
+                                            r[0], maybe_convert_from_base_unit(r[1], r[3], r[2]), new_value)
+                        elif r[0] in self._server_parameters \
+                                and not compare_values(r[3], r[2], r[1], self._server_parameters[r[0]]):
+                            # Check if any parameter was set back to the current pg_settings value
+                            # We can use pg_settings value here, as it is proved to be equal to new_value
+                            logger.info("Changed %s from '%s' to '%s'", r[0], self._server_parameters[r[0]], new_value)
+                            conf_changed = True
                 for param, value in changes.items():
                     if '.' in param:
-                        # Check that user-defined-paramters have changed (parameters with period in name)
+                        # Check that user-defined-parameters have changed (parameters with period in name)
                         if value is None or param not in self._server_parameters \
                                 or str(value) != str(self._server_parameters[param]):
-                            logger.info('Changed %s from %s to %s', param, self._server_parameters.get(param), value)
+                            logger.info("Changed %s from '%s' to '%s'",
+                                        param, self._server_parameters.get(param), value)
                             conf_changed = True
                     elif param in server_parameters:
                         logger.warning('Removing invalid parameter `%s` from postgresql.parameters', param)
@@ -1031,7 +1163,6 @@ class ConfigHandler(object):
                 ident_changed = self._config.get('pg_ident', []) != config['pg_ident']
 
         self._config = config
-        self._postgresql.set_pending_restart(pending_restart)
         self._server_parameters = server_parameters
         self._adjust_recovery_parameters()
         self._krbsrvname = config.get('krbsrvname')
@@ -1061,25 +1192,36 @@ class ConfigHandler(object):
             if self._postgresql.major_version >= 90500:
                 time.sleep(1)
                 try:
-                    pending_restart = (self._postgresql.query(
+                    settings_diff: CaseInsensitiveDict = CaseInsensitiveDict()
-                        'SELECT COUNT(*) FROM pg_catalog.pg_settings'
+                    for param, value, unit, vartype in self._postgresql.query(
-                        ' WHERE pg_catalog.lower(name) != ALL(%s) AND pending_restart',
+                            'SELECT name, pg_catalog.current_setting(name), unit, vartype FROM pg_catalog.pg_settings'
-                        [n.lower() for n in self._RECOVERY_PARAMETERS]).fetchone() or (0,))[0] > 0
+                            ' WHERE pg_catalog.lower(name) != ALL(%s) AND pending_restart',
-                    self._postgresql.set_pending_restart(pending_restart)
+                            [n.lower() for n in params_skip_changes]):
+                        new_value = self._postgresql.get_guc_value(param)
+                        new_value = '?' if new_value is None else new_value
+                        settings_diff[param] = get_param_diff(value, new_value, vartype, unit)
+                    external_change = {param: value for param, value in settings_diff.items()
+                                       if param not in param_diff or value != param_diff[param]}
+                    if external_change:
+                        logger.info("PostgreSQL configuration parameters requiring restart"
+                                    " (%s) seem to be changed bypassing Patroni config."
+                                    " Setting 'Pending restart' flag", ', '.join(external_change))
+                    param_diff = settings_diff
                 except Exception as e:
                     logger.warning('Exception %r when running query', e)
         else:
             logger.info('No PostgreSQL configuration items changed, nothing to reload.')
 
+        self._postgresql.set_pending_restart_reason(param_diff)
+
     def set_synchronous_standby_names(self, value: Optional[str]) -> Optional[bool]:
         """Updates synchronous_standby_names and reloads if necessary.
         :returns: True if value was updated."""
-        if value != self._synchronous_standby_names:
+        if value != self._server_parameters.get('synchronous_standby_names'):
             if value is None:
                 self._server_parameters.pop('synchronous_standby_names', None)
             else:
                 self._server_parameters['synchronous_standby_names'] = value
-            self._synchronous_standby_names = value
             if self._postgresql.state == 'running':
                 self.write_postgresql_conf()
                 self._postgresql.reload()
@@ -1114,6 +1256,7 @@ class ConfigHandler(object):
         data = self._postgresql.controldata()
         effective_configuration = self._server_parameters.copy()
 
+        param_diff = CaseInsensitiveDict()
         for name, cname in options_mapping.items():
             value = parse_int(effective_configuration[name])
             if cname not in data:
@@ -1123,7 +1266,10 @@ class ConfigHandler(object):
             cvalue = parse_int(data[cname])
             if cvalue is not None and value is not None and cvalue > value:
                 effective_configuration[name] = cvalue
-                self._postgresql.set_pending_restart(True)
+                logger.info("%s value in pg_controldata: %d, in the global configuration: %d."
+                            " pg_controldata value will be used. Setting 'Pending restart' flag", name, cvalue, value)
+                param_diff[name] = get_param_diff(cvalue, value)
+        self._postgresql.set_pending_restart_reason(param_diff)
 
         # If we are using custom bootstrap with PITR it could fail when values like max_connections
         # are increased, therefore we disable hot_standby if recovery_target_action == 'promote'.
@@ -1140,7 +1286,6 @@ class ConfigHandler(object):
 
             if disable_hot_standby:
                 effective_configuration['hot_standby'] = 'off'
-                self._postgresql.set_pending_restart(True)
 
         return effective_configuration
 

patroni/postgresql/connection.py

@@ -2,49 +2,158 @@ import logging
 
 from contextlib import contextmanager
 from threading import Lock
-from typing import Any, Dict, Generator, Union, TYPE_CHECKING
+from typing import Any, Dict, Iterator, List, Optional, Union, Tuple, TYPE_CHECKING
 if TYPE_CHECKING:  # pragma: no cover
-    from psycopg import Connection as Connection3, Cursor
+    from psycopg import Connection, Cursor
     from psycopg2 import connection, cursor
 
 from .. import psycopg
+from ..exceptions import PostgresConnectionException
 
 logger = logging.getLogger(__name__)
 
 
-class Connection(object):
+class NamedConnection:
+    """Helper class to manage ``psycopg`` connections from Patroni to PostgreSQL.
+
+    :ivar server_version: PostgreSQL version in integer format where we are connected to.
+    """
+
     server_version: int
 
-    def __init__(self) -> None:
+    def __init__(self, pool: 'ConnectionPool', name: str, kwargs_override: Optional[Dict[str, Any]]) -> None:
-        self._lock = Lock()
+        """Create an instance of :class:`NamedConnection` class.
+
+        :param pool: reference to a :class:`ConnectionPool` object.
+        :param name: name of the connection.
+        :param kwargs_override: :class:`dict` object with connection parameters that should be
+                                different from default values provided by connection *pool*.
+        """
+        self._pool = pool
+        self._name = name
+        self._kwargs_override = kwargs_override or {}
+        self._lock = Lock()  # used to make sure that only one connection to postgres is established
         self._connection = None
-        self._cursor_holder = None
 
-    def set_conn_kwargs(self, conn_kwargs: Dict[str, Any]) -> None:
+    @property
-        self._conn_kwargs = conn_kwargs
+    def _conn_kwargs(self) -> Dict[str, Any]:
+        """Connection parameters for this :class:`NamedConnection`."""
+        return {**self._pool.conn_kwargs, **self._kwargs_override, 'application_name': f'Patroni {self._name}'}
 
-    def get(self) -> Union['connection', 'Connection3[Any]']:
+    def get(self) -> Union['connection', 'Connection[Any]']:
+        """Get ``psycopg``/``psycopg2`` connection object.
+
+        .. note::
+            Opens a new connection if necessary.
+
+        :returns: ``psycopg`` or ``psycopg2`` connection object.
+        """
         with self._lock:
             if not self._connection or self._connection.closed != 0:
+                logger.info("establishing a new patroni %s connection to postgres", self._name)
                 self._connection = psycopg.connect(**self._conn_kwargs)
                 self.server_version = getattr(self._connection, 'server_version', 0)
         return self._connection
 
-    def cursor(self) -> Union['cursor', 'Cursor[Any]']:
+    def query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]:
-        if not self._cursor_holder or self._cursor_holder.closed or self._cursor_holder.connection.closed != 0:
+        """Execute a query with parameters and optionally returns a response.
-            logger.info("establishing a new patroni connection to the postgres cluster")
-            self._cursor_holder = self.get().cursor()
-        return self._cursor_holder
 
-    def close(self) -> None:
+        :param sql: SQL statement to execute.
+        :param params: parameters to pass.
+
+        :returns: a query response as a list of tuples if there is any.
+        :raises:
+            :exc:`~psycopg.Error` if had issues while executing *sql*.
+
+            :exc:`~patroni.exceptions.PostgresConnectionException`: if had issues while connecting to the database.
+        """
+        cursor = None
+        try:
+            with self.get().cursor() as cursor:
+                cursor.execute(sql.encode('utf-8'), params or None)
+                return cursor.fetchall() if cursor.rowcount and cursor.rowcount > 0 else []
+        except psycopg.Error as exc:
+            if cursor and cursor.connection.closed == 0:
+                # When connected via unix socket, psycopg2 can't recoginze 'connection lost' and leaves
+                # `self._connection.closed == 0`, but the generic exception is raised. It doesn't make
+                # sense to continue with existing connection and we will close it, to avoid its reuse.
+                if type(exc) in (psycopg.DatabaseError, psycopg.OperationalError):
+                    self.close()
+                else:
+                    raise exc
+            raise PostgresConnectionException('connection problems') from exc
+
+    def close(self, silent: bool = False) -> bool:
+        """Close the psycopg connection to postgres.
+
+        :param silent: whether the method should not write logs.
+
+        :returns: ``True`` if ``psycopg`` connection was closed, ``False`` otherwise.``
+        """
+        ret = False
         if self._connection and self._connection.closed == 0:
             self._connection.close()
-            logger.info("closed patroni connection to the postgresql cluster")
+            if not silent:
-        self._cursor_holder = self._connection = None
+                logger.info("closed patroni %s connection to postgres", self._name)
+            ret = True
+        self._connection = None
+        return ret
+
+
+class ConnectionPool:
+    """Helper class to manage named connections from Patroni to PostgreSQL.
+
+    The instance keeps named :class:`NamedConnection` objects and parameters that must be used for new connections.
+    """
+
+    def __init__(self) -> None:
+        """Create an instance of :class:`ConnectionPool` class."""
+        self._lock = Lock()
+        self._connections: Dict[str, NamedConnection] = {}
+        self._conn_kwargs: Dict[str, Any] = {}
+
+    @property
+    def conn_kwargs(self) -> Dict[str, Any]:
+        """Connection parameters that must be used for new ``psycopg`` connections."""
+        with self._lock:
+            return self._conn_kwargs.copy()
+
+    @conn_kwargs.setter
+    def conn_kwargs(self, value: Dict[str, Any]) -> None:
+        """Set new connection parameters.
+
+        :param value: :class:`dict` object with connection parameters.
+        """
+        with self._lock:
+            self._conn_kwargs = value
+
+    def get(self, name: str, kwargs_override: Optional[Dict[str, Any]] = None) -> NamedConnection:
+        """Get a new named :class:`NamedConnection` object from the pool.
+
+        .. note::
+            Creates a new :class:`NamedConnection` object if it doesn't yet exist in the pool.
+
+        :param name: name of the connection.
+        :param kwargs_override: :class:`dict` object with connection parameters that should be
+                                different from default values provided by :attr:`conn_kwargs`.
+
+        :returns: :class:`NamedConnection` object.
+        """
+        with self._lock:
+            if name not in self._connections:
+                self._connections[name] = NamedConnection(self, name, kwargs_override)
+        return self._connections[name]
+
+    def close(self) -> None:
+        """Close all named connections from Patroni to PostgreSQL registered in the pool."""
+        with self._lock:
+            closed_connections = [conn.close(True) for conn in self._connections.values()]
+            if any(closed_connections):
+                logger.info("closed patroni connections to postgres")
 
 
 @contextmanager
-def get_connection_cursor(**kwargs: Any) -> Generator[Union['cursor', 'Cursor[Any]'], None, None]:
+def get_connection_cursor(**kwargs: Any) -> Iterator[Union['cursor', 'Cursor[Any]']]:
     conn = psycopg.connect(**kwargs)
     with conn.cursor() as cur:
         yield cur

patroni/postgresql/mpp/__init__.py

@@ -0,0 +1,315 @@
+"""Abstract classes for MPP handler.
+
+MPP stands for Massively Parallel Processing, and Citus belongs to this architecture. Currently, Citus is the only
+supported MPP cluster. However, we may consider adapting other databases such as TimescaleDB, GPDB, etc. into Patroni.
+"""
+import abc
+
+from typing import Any, Dict, Iterator, Optional, Union, Tuple, Type, TYPE_CHECKING
+
+from ...dcs import Cluster
+from ...dynamic_loader import iter_classes
+from ...exceptions import PatroniException
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .. import Postgresql
+    from ...config import Config
+
+
+class AbstractMPP(abc.ABC):
+    """An abstract class which should be passed to :class:`AbstractDCS`.
+
+    .. note::
+        We create :class:`AbstractMPP` and :class:`AbstractMPPHandler` to solve the chicken-egg initialization problem.
+        When initializing DCS, we dynamically create an object implementing :class:`AbstractMPP`, later this object is
+        used to instantiate an object implementing :class:`AbstractMPPHandler`.
+    """
+
+    group_re: Any  # re.Pattern[str]
+
+    def __init__(self, config: Dict[str, Union[str, int]]) -> None:
+        """Init method for :class:`AbstractMPP`.
+
+        :param config: configuration of MPP section.
+        """
+        self._config = config
+
+    def is_enabled(self) -> bool:
+        """Check if MPP is enabled for a given MPP.
+
+        .. note::
+            We just check that the :attr:`_config` object isn't empty and expect
+            it to be empty only in case of :class:`Null`.
+
+        :returns: ``True`` if MPP is enabled, otherwise ``False``.
+        """
+        return bool(self._config)
+
+    @staticmethod
+    @abc.abstractmethod
+    def validate_config(config: Any) -> bool:
+        """Check whether provided config is good for a given MPP.
+
+        :param config: configuration of MPP section.
+
+        :returns: ``True`` is config passes validation, otherwise ``False``.
+        """
+
+    @property
+    @abc.abstractmethod
+    def group(self) -> Any:
+        """The group for a given MPP implementation."""
+
+    @property
+    @abc.abstractmethod
+    def coordinator_group_id(self) -> Any:
+        """The group id of the coordinator PostgreSQL cluster."""
+
+    @property
+    def type(self) -> str:
+        """The type of the MPP cluster.
+
+        :returns: A string representation of the type of a given MPP implementation.
+        """
+        for base in self.__class__.__bases__:
+            if not base.__name__.startswith('Abstract'):
+                return base.__name__
+        return self.__class__.__name__
+
+    @property
+    def k8s_group_label(self):
+        """Group label used for kubernetes DCS of the MPP cluster.
+
+        :returns: A string representation of the k8s group label of a given MPP implementation.
+        """
+        return self.type.lower() + '-group'
+
+    def is_coordinator(self) -> bool:
+        """Check whether this node is running in the coordinator PostgreSQL cluster.
+
+        :returns: ``True`` if MPP is enabled and the group id of this node
+                  matches with the :attr:`coordinator_group_id`, otherwise ``False``.
+        """
+        return self.is_enabled() and self.group == self.coordinator_group_id
+
+    def is_worker(self) -> bool:
+        """Check whether this node is running as a MPP worker PostgreSQL cluster.
+
+        :returns: ``True`` if MPP is enabled and this node is known to be not running
+                  as the coordinator PostgreSQL cluster, otherwise ``False``.
+        """
+        return self.is_enabled() and not self.is_coordinator()
+
+    def _get_handler_cls(self) -> Iterator[Type['AbstractMPPHandler']]:
+        """Find Handler classes inherited from a class type of this object.
+
+        :yields: handler classes for this object.
+        """
+        for cls in self.__class__.__subclasses__():
+            if issubclass(cls, AbstractMPPHandler) and cls.__name__.startswith(self.__class__.__name__):
+                yield cls
+
+    def get_handler_impl(self, postgresql: 'Postgresql') -> 'AbstractMPPHandler':
+        """Find and instantiate Handler implementation of this object.
+
+        :param postgresql: a reference to :class:`Postgresql` object.
+
+        :raises:
+            :exc:`PatroniException`: if the Handler class haven't been found.
+
+        :returns: an instantiated class that implements Handler for this object.
+        """
+        for cls in self._get_handler_cls():
+            return cls(postgresql, self._config)
+        raise PatroniException(f'Failed to initialize {self.__class__.__name__}Handler object')
+
+
+class AbstractMPPHandler(AbstractMPP):
+    """An abstract class which defines interfaces that should be implemented by real handlers."""
+
+    def __init__(self, postgresql: 'Postgresql', config: Dict[str, Union[str, int]]) -> None:
+        """Init method for :class:`AbstractMPPHandler`.
+
+        :param postgresql: a reference to :class:`Postgresql` object.
+        :param config: configuration of MPP section.
+        """
+        super().__init__(config)
+        self._postgresql = postgresql
+
+    @abc.abstractmethod
+    def handle_event(self, cluster: Cluster, event: Dict[str, Any]) -> None:
+        """Handle an event sent from a worker node.
+
+        :param cluster: the currently known cluster state from DCS.
+        :param event: the event to be handled.
+        """
+
+    @abc.abstractmethod
+    def sync_meta_data(self, cluster: Cluster) -> None:
+        """Sync meta data on the coordinator.
+
+        :param cluster: the currently known cluster state from DCS.
+        """
+
+    @abc.abstractmethod
+    def on_demote(self) -> None:
+        """On demote handler.
+
+        Is called when the primary was demoted.
+        """
+
+    @abc.abstractmethod
+    def schedule_cache_rebuild(self) -> None:
+        """Cache rebuild handler.
+
+        Is called to notify handler that it has to refresh its metadata cache from the database.
+        """
+
+    @abc.abstractmethod
+    def bootstrap(self) -> None:
+        """Bootstrap handler.
+
+        Is called when the new cluster is initialized (through ``initdb`` or a custom bootstrap method).
+        """
+
+    @abc.abstractmethod
+    def adjust_postgres_gucs(self, parameters: Dict[str, Any]) -> None:
+        """Adjust GUCs in the current PostgreSQL configuration.
+
+        :param parameters: dictionary of GUCs, with key as GUC name and the corresponding value as current GUC value.
+        """
+
+    @abc.abstractmethod
+    def ignore_replication_slot(self, slot: Dict[str, str]) -> bool:
+        """Check whether provided replication *slot* existing in the database should not be removed.
+
+        .. note::
+            MPP database may create replication slots for its own use, for example to migrate data between workers
+            using logical replication, and we don't want to suddenly drop them.
+
+        :param slot: dictionary containing the replication slot settings, like ``name``, ``database``, ``type``, and
+                     ``plugin``.
+
+        :returns: ``True`` if the replication slots should not be removed, otherwise ``False``.
+        """
+
+
+class Null(AbstractMPP):
+    """Dummy implementation of :class:`AbstractMPP`."""
+
+    def __init__(self) -> None:
+        """Init method for :class:`Null`."""
+        super().__init__({})
+
+    @staticmethod
+    def validate_config(config: Any) -> bool:
+        """Check whether provided config is good for :class:`Null`.
+
+        :returns: always ``True``.
+        """
+        return True
+
+    @property
+    def group(self) -> None:
+        """The group for :class:`Null`.
+
+        :returns: always ``None``.
+        """
+        return None
+
+    @property
+    def coordinator_group_id(self) -> None:
+        """The group id of the coordinator PostgreSQL cluster.
+
+        :returns: always ``None``.
+        """
+        return None
+
+
+class NullHandler(Null, AbstractMPPHandler):
+    """Dummy implementation of :class:`AbstractMPPHandler`."""
+
+    def __init__(self, postgresql: 'Postgresql', config: Dict[str, Union[str, int]]) -> None:
+        """Init method for :class:`NullHandler`.
+
+        :param postgresql: a reference to :class:`Postgresql` object.
+        :param config: configuration of MPP section.
+        """
+        AbstractMPPHandler.__init__(self, postgresql, config)
+
+    def handle_event(self, cluster: Cluster, event: Dict[str, Any]) -> None:
+        """Handle an event sent from a worker node.
+
+        :param cluster: the currently known cluster state from DCS.
+        :param event: the event to be handled.
+        """
+
+    def sync_meta_data(self, cluster: Cluster) -> None:
+        """Sync meta data on the coordinator.
+
+        :param cluster: the currently known cluster state from DCS.
+        """
+
+    def on_demote(self) -> None:
+        """On demote handler.
+
+        Is called when the primary was demoted.
+        """
+
+    def schedule_cache_rebuild(self) -> None:
+        """Cache rebuild handler.
+
+        Is called to notify handler that it has to refresh its metadata cache from the database.
+        """
+
+    def bootstrap(self) -> None:
+        """Bootstrap handler.
+
+        Is called when the new cluster is initialized (through ``initdb`` or a custom bootstrap method).
+        """
+
+    def adjust_postgres_gucs(self, parameters: Dict[str, Any]) -> None:
+        """Adjust GUCs in the current PostgreSQL configuration.
+
+        :param parameters: dictionary of GUCs, with key as GUC name and corresponding value as current GUC value.
+        """
+
+    def ignore_replication_slot(self, slot: Dict[str, str]) -> bool:
+        """Check whether provided replication *slot* existing in the database should not be removed.
+
+        .. note::
+            MPP database may create replication slots for its own use, for example to migrate data between workers
+            using logical replication, and we don't want to suddenly drop them.
+
+        :param slot: dictionary containing the replication slot settings, like ``name``, ``database``, ``type``, and
+                     ``plugin``.
+
+        :returns: always ``False``.
+        """
+        return False
+
+
+def iter_mpp_classes(
+        config: Optional[Union['Config', Dict[str, Any]]] = None
+) -> Iterator[Tuple[str, Type[AbstractMPP]]]:
+    """Attempt to import MPP modules that are present in the given configuration.
+
+    :param config: configuration information with possible MPP names as keys. If given, only attempt to import MPP
+                   modules defined in the configuration. Else, if ``None``, attempt to import any supported MPP module.
+
+    :yields: tuples, each containing the module ``name`` and the imported MPP class object.
+    """
+    yield from iter_classes(__package__, AbstractMPP, config)
+
+
+def get_mpp(config: Union['Config', Dict[str, Any]]) -> AbstractMPP:
+    """Attempt to load and instantiate a MPP module from known available implementations.
+
+    :param config: object or dictionary with Patroni configuration.
+
+    :returns: The successfully loaded MPP or fallback to :class:`Null`.
+    """
+    for name, mpp_class in iter_mpp_classes(config):
+        if mpp_class.validate_config(config[name]):
+            return mpp_class(config[name])
+    return Null()

patroni/postgresql/mpp/citus.py

@@ -6,15 +6,15 @@ from threading import Condition, Event, Thread
 from urllib.parse import urlparse
 from typing import Any, Dict, List, Optional, Union, Tuple, TYPE_CHECKING
 
-from .connection import Connection
+from . import AbstractMPP, AbstractMPPHandler
-from ..dcs import CITUS_COORDINATOR_GROUP_ID, Cluster
+from ...dcs import Cluster
-from ..psycopg import connect, quote_ident
+from ...psycopg import connect, quote_ident, ProgrammingError
+from ...utils import parse_int
 
 if TYPE_CHECKING:  # pragma: no cover
-    from psycopg import Cursor
+    from .. import Postgresql
-    from psycopg2 import cursor
-    from . import Postgresql
 
+CITUS_COORDINATOR_GROUP_ID = 0
 CITUS_SLOT_NAME_RE = re.compile(r'^citus_shard_(move|split)_slot(_[1-9][0-9]*){2,3}$')
 logger = logging.getLogger(__name__)
 
@@ -66,14 +66,49 @@ class PgDistNode(object):
         return str(self)
 
 
-class CitusHandler(Thread):
+class Citus(AbstractMPP):
 
-    def __init__(self, postgresql: 'Postgresql', config: Optional[Dict[str, Union[str, int]]]) -> None:
+    group_re = re.compile('^(0|[1-9][0-9]*)$')
-        super(CitusHandler, self).__init__()
+
+    @staticmethod
+    def validate_config(config: Union[Any, Dict[str, Union[str, int]]]) -> bool:
+        """Check whether provided config is good for a given MPP.
+
+        :param config: configuration of ``citus`` MPP section.
+
+        :returns: ``True`` is config passes validation, otherwise ``False``.
+        """
+        return isinstance(config, dict) \
+            and isinstance(config.get('database'), str) \
+            and parse_int(config.get('group')) is not None
+
+    @property
+    def group(self) -> int:
+        """The group of this Citus node."""
+        return int(self._config['group'])
+
+    @property
+    def coordinator_group_id(self) -> int:
+        """The group id of the Citus coordinator PostgreSQL cluster."""
+        return CITUS_COORDINATOR_GROUP_ID
+
+
+class CitusHandler(Citus, AbstractMPPHandler, Thread):
+    """Define the interfaces for handling an underlying Citus cluster."""
+
+    def __init__(self, postgresql: 'Postgresql', config: Dict[str, Union[str, int]]) -> None:
+        """"Initialize a new instance of :class:`CitusHandler`.
+
+        :param postgresql: the Postgres node.
+        :param config: the ``citus`` MPP config section.
+        """
+        Thread.__init__(self)
+        AbstractMPPHandler.__init__(self, postgresql, config)
         self.daemon = True
-        self._postgresql = postgresql
+        if config:
-        self._config = config
+            self._connection = postgresql.connection_pool.get(
-        self._connection = Connection()
+                'citus', {'dbname': config['database'],
+                          'options': '-c statement_timeout=0 -c idle_in_transaction_session_timeout=0'})
         self._pg_dist_node: Dict[int, PgDistNode] = {}  # Cache of pg_dist_node: {groupid: PgDistNode()}
         self._tasks: List[PgDistNode] = []  # Requests to change pg_dist_node, every task is a `PgDistNode`
         self._in_flight: Optional[PgDistNode] = None  # Reference to the `PgDistNode` being changed in a transaction
@@ -81,40 +116,25 @@ class CitusHandler(Thread):
         self._condition = Condition()  # protects _pg_dist_node, _tasks, _in_flight, and _schedule_load_pg_dist_node
         self.schedule_cache_rebuild()
 
-    def is_enabled(self) -> bool:
-        return isinstance(self._config, dict)
-
-    def group(self) -> Optional[int]:
-        return int(self._config['group']) if isinstance(self._config, dict) else None
-
-    def is_coordinator(self) -> bool:
-        return self.is_enabled() and self.group() == CITUS_COORDINATOR_GROUP_ID
-
-    def is_worker(self) -> bool:
-        return self.is_enabled() and not self.is_coordinator()
-
-    def set_conn_kwargs(self, kwargs: Dict[str, Any]) -> None:
-        if isinstance(self._config, dict):  # self.is_enabled():
-            kwargs.update({'dbname': self._config['database'],
-                           'options': '-c statement_timeout=0 -c idle_in_transaction_session_timeout=0'})
-            self._connection.set_conn_kwargs(kwargs)
-
     def schedule_cache_rebuild(self) -> None:
+        """Cache rebuild handler.
+
+        Is called to notify handler that it has to refresh its metadata cache from the database.
+        """
         with self._condition:
             self._schedule_load_pg_dist_node = True
 
     def on_demote(self) -> None:
         with self._condition:
             self._pg_dist_node.clear()
-            self._tasks[:] = []
+            empty_tasks: List[PgDistNode] = []
+            self._tasks[:] = empty_tasks
             self._in_flight = None
 
-    def query(self, sql: str, *params: Any) -> Union['Cursor[Any]', 'cursor']:
+    def query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]:
         try:
             logger.debug('query(%s, %s)', sql, params)
-            cursor = self._connection.cursor()
+            return self._connection.query(sql, *params)
-            cursor.execute(sql.encode('utf-8'), params or None)
-            return cursor
         except Exception as e:
             logger.error('Exception when executing query "%s", (%s): %r', sql, params, e)
             self._connection.close()
@@ -132,17 +152,17 @@ class CitusHandler(Thread):
             self._schedule_load_pg_dist_node = False
 
         try:
-            cursor = self.query("SELECT nodeid, groupid, nodename, nodeport, noderole"
+            rows = self.query("SELECT nodeid, groupid, nodename, nodeport, noderole"
-                                " FROM pg_catalog.pg_dist_node WHERE noderole = 'primary'")
+                              " FROM pg_catalog.pg_dist_node WHERE noderole = 'primary'")
         except Exception:
             return False
 
         with self._condition:
-            self._pg_dist_node = {r[1]: PgDistNode(r[1], r[2], r[3], 'after_promote', r[0]) for r in cursor}
+            self._pg_dist_node = {r[1]: PgDistNode(r[1], r[2], r[3], 'after_promote', r[0]) for r in rows}
         return True
 
-    def sync_pg_dist_node(self, cluster: Cluster) -> None:
+    def sync_meta_data(self, cluster: Cluster) -> None:
-        """Maintain the `pg_dist_node` from the coordinator leader every heartbeat loop.
+        """Maintain the ``pg_dist_node`` from the coordinator leader every heartbeat loop.
 
         We can't always rely on REST API calls from worker nodes in order
         to maintain `pg_dist_node`, therefore at least once per heartbeat
@@ -174,11 +194,13 @@ class CitusHandler(Thread):
         """Returns the tuple(i, task), where `i` - is the task index in the self._tasks list
 
         Tasks are picked by following priorities:
+
         1. If there is already a transaction in progress, pick a task
            that that will change already affected worker primary.
         2. If the coordinator address should be changed - pick a task
            with group=0 (coordinators are always in group 0).
-        3. Pick a task that is the oldest (first from the self._tasks)"""
+        3. Pick a task that is the oldest (first from the self._tasks)
+        """
 
         with self._condition:
             if self._in_flight:
@@ -209,10 +231,8 @@ class CitusHandler(Thread):
             self.query('SELECT pg_catalog.citus_update_node(%s, %s, %s, true, %s)',
                        task.nodeid, task.host, task.port, task.cooldown)
         elif task.event != 'before_demote':
-            row = self.query("SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default')",
+            task.nodeid = self.query("SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default')",
-                             task.host, task.port, task.group).fetchone()
+                                     task.host, task.port, task.group)[0][0]
-            if row is not None:
-                task.nodeid = row[0]
 
     def process_task(self, task: PgDistNode) -> bool:
         """Updates a single row in `pg_dist_node` table, optionally in a transaction.
@@ -303,16 +323,16 @@ class CitusHandler(Thread):
         with self._condition:
             i = self.find_task_by_group(task.group)
 
-            # The `PgDistNode.timeout` == None is an indicator that it was scheduled from the sync_pg_dist_node().
+            # The `PgDistNode.timeout` == None is an indicator that it was scheduled from the sync_meta_data().
             if task.timeout is None:
                 # We don't want to override the already existing task created from REST API.
                 if i is not None and self._tasks[i].timeout is not None:
                     return False
 
                 # There is a little race condition with tasks created from REST API - the call made "before" the member
-                # key is updated in DCS. Therefore it is possible that :func:`sync_pg_dist_node` will try to create a
+                # key is updated in DCS. Therefore it is possible that :func:`sync_meta_data` will try to create a task
-                # task based on the outdated values of "state"/"role". To solve it we introduce an artificial timeout.
+                # based on the outdated values of "state"/"role". To solve it we introduce an artificial timeout.
-                # Only when the timeout is reached new tasks could be scheduled from sync_pg_dist_node()
+                # Only when the timeout is reached new tasks could be scheduled from sync_meta_data()
                 if self._in_flight and self._in_flight.group == task.group and self._in_flight.timeout is not None\
                         and self._in_flight.deadline > time.time():
                     return False
@@ -360,17 +380,23 @@ class CitusHandler(Thread):
             task.wait()
 
     def bootstrap(self) -> None:
-        if not isinstance(self._config, dict):  # self.is_enabled()
+        """Bootstrap handler.
-            return
 
-        conn_kwargs = self._postgresql.config.local_connect_kwargs
+        Is called when the new cluster is initialized (through ``initdb`` or a custom bootstrap method).
-        conn_kwargs['options'] = '-c synchronous_commit=local -c statement_timeout=0'
+        """
+        conn_kwargs = {**self._postgresql.connection_pool.conn_kwargs,
+                       'options': '-c synchronous_commit=local -c statement_timeout=0'}
         if self._config['database'] != self._postgresql.database:
             conn = connect(**conn_kwargs)
             try:
                 with conn.cursor() as cur:
                     cur.execute('CREATE DATABASE {0}'.format(
                         quote_ident(self._config['database'], conn)).encode('utf-8'))
+            except ProgrammingError as exc:
+                if exc.diag.sqlstate == '42P04':  # DuplicateDatabase
+                    logger.debug('Exception when creating database: %r', exc)
+                else:
+                    raise exc
             finally:
                 conn.close()
 
@@ -378,7 +404,7 @@ class CitusHandler(Thread):
         conn = connect(**conn_kwargs)
         try:
             with conn.cursor() as cur:
-                cur.execute('CREATE EXTENSION citus')
+                cur.execute('CREATE EXTENSION IF NOT EXISTS citus')
 
                 superuser = self._postgresql.config.superuser
                 params = {k: superuser[k] for k in ('password', 'sslcert', 'sslkey') if k in superuser}
@@ -395,9 +421,10 @@ class CitusHandler(Thread):
             conn.close()
 
     def adjust_postgres_gucs(self, parameters: Dict[str, Any]) -> None:
-        if not self.is_enabled():
+        """Adjust GUCs in the current PostgreSQL configuration.
-            return
 
+        :param parameters: dictionary of GUCs, with key as GUC name and the corresponding value as current GUC value.
+        """
         # citus extension must be on the first place in shared_preload_libraries
         shared_preload_libraries = list(filter(
             lambda el: el and el != 'citus',
@@ -405,15 +432,28 @@ class CitusHandler(Thread):
         parameters['shared_preload_libraries'] = ','.join(['citus'] + shared_preload_libraries)
 
         # if not explicitly set Citus overrides max_prepared_transactions to max_connections*2
-        if parameters.get('max_prepared_transactions') == 0:
+        if parameters['max_prepared_transactions'] == 0:
             parameters['max_prepared_transactions'] = parameters['max_connections'] * 2
 
         # Resharding in Citus implemented using logical replication
         parameters['wal_level'] = 'logical'
 
+        # Sometimes Citus needs to connect to the local postgres. We will do it the same way as Patroni does.
+        parameters['citus.local_hostname'] = self._postgresql.connection_pool.conn_kwargs.get('host', 'localhost')
+
     def ignore_replication_slot(self, slot: Dict[str, str]) -> bool:
-        if isinstance(self._config, dict) and self._postgresql.is_leader() and\
+        """Check whether provided replication *slot* existing in the database should not be removed.
-                slot['type'] == 'logical' and slot['database'] == self._config['database']:
+
+        .. note::
+            MPP database may create replication slots for its own use, for example to migrate data between workers
+            using logical replication, and we don't want to suddenly drop them.
+
+        :param slot: dictionary containing the replication slot settings, like ``name``, ``database``, ``type``, and
+                     ``plugin``.
+
+        :returns: ``True`` if the replication slots should not be removed, otherwise ``False``.
+        """
+        if self._postgresql.is_primary() and slot['type'] == 'logical' and slot['database'] == self._config['database']:
             m = CITUS_SLOT_NAME_RE.match(slot['name'])
             return bool(m and {'move': 'pgoutput', 'split': 'citus'}.get(m.group(1)) == slot['plugin'])
         return False

patroni/postgresql/postmaster.py

@@ -176,7 +176,7 @@ class PostmasterProcess(psutil.Process):
             return not self.is_running()
 
     def wait_for_user_backends_to_close(self, stop_timeout: Optional[float]) -> None:
-        # These regexps are cross checked against versions PostgreSQL 9.1 .. 15
+        # These regexps are cross checked against versions PostgreSQL 9.1 .. 16
         aux_proc_re = re.compile("(?:postgres:)( .*:)? (?:(?:archiver|startup|autovacuum launcher|autovacuum worker|"
                                  "checkpointer|logger|stats collector|wal receiver|wal writer|writer)(?: process  )?|"
                                  "walreceiver|wal sender process|walsender|walwriter|background writer|"

patroni/postgresql/rewind.py

@@ -101,12 +101,26 @@ class Rewind(object):
             return 'not accessible or not healty'
 
     def _get_checkpoint_end(self, timeline: int, lsn: int) -> int:
-        """The checkpoint record size in WAL depends on postgres major version and platform (memory alignment).
+        """Get the end of checkpoint record from WAL.
-        Hence, the only reliable way to figure out where it ends, read the record from file with the help of pg_waldump
-        and parse the output. We are trying to read two records, and expect that it will fail to read the second one:
-        `pg_waldump: fatal: error in WAL record at 0/182E220: invalid record length at 0/182E298: wanted 24, got 0`
-        The error message contains information about LSN of the next record, which is exactly where checkpoint ends."""
 
+        .. note::
+            The checkpoint record size in WAL depends on postgres major version and platform (memory alignment).
+            Hence, the only reliable way to figure out where it ends, is to read the record from file with the
+            help of ``pg_waldump`` and parse the output.
+
+            We are trying to read two records, and expect that it will fail to read the second record with message:
+
+                fatal: error in WAL record at 0/182E220: invalid record length at 0/182E298: wanted 24, got 0; or
+
+                fatal: error in WAL record at 0/182E220: invalid record length at 0/182E298: expected at least 24, got 0
+
+            The error message contains information about LSN of the next record, which is exactly where checkpoint ends.
+
+        :param timeline: the checkpoint *timeline* from ``pg_controldata``.
+        :param lsn: the checkpoint *location* as :class:`int` from ``pg_controldata``.
+
+        :returns: the end of checkpoint record as :class:`int` or ``0`` if failed to parse ``pg_waldump`` output.
+        """
         lsn8 = format_lsn(lsn, True)
         lsn_str = format_lsn(lsn)
         out, err = self._postgresql.waldump(timeline, lsn_str, 2)
@@ -117,12 +131,17 @@ class Rewind(object):
 
             if len(out) == 1 and len(err) == 1 and ', lsn: {0}, prev '.format(lsn8) in out[0] and pattern in err[0]:
                 i = err[0].find(pattern) + len(pattern)
-                j = err[0].find(": wanted ", i)
+                # Message format depends on the major version:
-                if j > -1:
+                # * expected at least -- starting from v16
-                    try:
+                # * wanted -- before v16
-                        return parse_lsn(err[0][i:j])
+                # We will simply check all possible combinations.
-                    except Exception as e:
+                for pattern in (': expected at least ', ': wanted '):
-                        logger.error('Failed to parse lsn %s: %r', err[0][i:j], e)
+                    j = err[0].find(pattern, i)
+                    if j > -1:
+                        try:
+                            return parse_lsn(err[0][i:j])
+                        except Exception as e:
+                            logger.error('Failed to parse lsn %s: %r', err[0][i:j], e)
             logger.error('Failed to parse pg_%sdump output', self._postgresql.wal_name)
             logger.error(' stdout=%s', '\n'.join(out))
             logger.error(' stderr=%s', '\n'.join(err))
@@ -158,7 +177,7 @@ class Rewind(object):
     def _get_local_timeline_lsn(self) -> Tuple[Optional[bool], Optional[int], Optional[int]]:
         if self._postgresql.is_running():  # if postgres is running - get timeline from replication connection
             in_recovery = True
-            timeline = self._postgresql.received_timeline() or self._postgresql.get_replica_timeline()
+            timeline = self._postgresql.get_replica_timeline()
             lsn = self._postgresql.replayed_location()
         else:  # otherwise analyze pg_controldata output
             in_recovery, timeline, lsn = self._get_local_timeline_lsn_from_controldata()
@@ -190,9 +209,10 @@ class Rewind(object):
         ret = member.conn_kwargs(auth)
         if not ret.get('dbname'):
             ret['dbname'] = self._postgresql.database
-        # Add target_session_attrs in case more than one hostname is specified
+        # Add target_session_attrs to make sure we hit the primary.
-        # (libpq client-side failover) making sure we hit the primary
+        # It is not strictly necessary for starting from PostgreSQL v14, which made it possible
-        if 'target_session_attrs' not in ret and self._postgresql.major_version >= 100000:
+        # to rewind from standby, but doing it from the real primary is always safer.
+        if self._postgresql.major_version >= 100000:
             ret['target_session_attrs'] = 'read-write'
         return ret
 
@@ -280,7 +300,7 @@ class Rewind(object):
         """After promote issue a CHECKPOINT from a new thread and asynchronously check the result.
         In case if CHECKPOINT failed, just check that timeline in pg_control was updated."""
 
-        if self._state == REWIND_STATUS.INITIAL and self._postgresql.is_leader():
+        if self._state != REWIND_STATUS.CHECKPOINT and self._postgresql.is_primary():
             with self._checkpoint_task_lock:
                 if self._checkpoint_task:
                     with self._checkpoint_task:
@@ -370,7 +390,7 @@ class Rewind(object):
                 # it is the author of archive_command, who is responsible
                 # for not overriding the WALs already present in archive
                 logger.info('Trying to archive %s: %s', wal, cmd)
-                if self._postgresql.cancellable.call(shlex.split(cmd)) == 0:
+                if self._postgresql.cancellable.call([cmd], shell=True) == 0:
                     new_name = os.path.join(status_dir, wal + '.done')
                     try:
                         shutil.move(old_name, new_name)

patroni/postgresql/slots.py

@@ -1,16 +1,23 @@
+"""Replication slot handling.
+
+Provides classes for the creation, monitoring, management and synchronisation of PostgreSQL replication slots.
+"""
+
 import logging
 import os
 import shutil
-
 from collections import defaultdict
 from contextlib import contextmanager
 from threading import Condition, Thread
-from typing import Any, Dict, Generator, List, Optional, Union, Tuple, TYPE_CHECKING
+from typing import Any, Dict, Iterator, List, Optional, Union, Tuple, TYPE_CHECKING, Collection
 
 from .connection import get_connection_cursor
 from .misc import format_lsn, fsync_dir
+from .. import global_config
 from ..dcs import Cluster, Leader
+from ..file_perm import pg_perm
 from ..psycopg import OperationalError
+from ..tags import Tags
 
 if TYPE_CHECKING:  # pragma: no cover
     from psycopg import Cursor
@@ -42,9 +49,17 @@ def compare_slots(s1: Dict[str, Any], s2: Dict[str, Any], dbid: str = 'database'
 
 
 class SlotsAdvanceThread(Thread):
+    """Daemon process :class:``Thread`` object for advancing logical replication slots on replicas.
+
+    This ensures that slot advancing queries sent to postgres do not block the main loop.
+    """
 
     def __init__(self, slots_handler: 'SlotsHandler') -> None:
-        super(SlotsAdvanceThread, self).__init__()
+        """Create and start a new thread for handling slot advance queries.
+
+        :param slots_handler: The calling class instance for reference to slot information attributes.
+        """
+        super().__init__()
         self.daemon = True
         self._slots_handler = slots_handler
 
@@ -58,6 +73,13 @@ class SlotsAdvanceThread(Thread):
         self.start()
 
     def sync_slot(self, cur: Union['cursor', 'Cursor[Any]'], database: str, slot: str, lsn: int) -> None:
+        """Execute a ``pg_replication_slot_advance`` query and store success for scheduled synchronisation task.
+
+        :param cur: database connection cursor.
+        :param database: name of the database associated with the slot.
+        :param slot: name of the slot to be synchronised.
+        :param lsn: last known LSN position
+        """
         failed = copy = False
         try:
             cur.execute("SELECT pg_catalog.pg_replication_slot_advance(%s, %s)", (slot, format_lsn(lsn)))
@@ -79,6 +101,11 @@ class SlotsAdvanceThread(Thread):
                     self._scheduled.pop(database)
 
     def sync_slots_in_database(self, database: str, slots: List[str]) -> None:
+        """Synchronise slots for a single database.
+
+        :param database: name of the database.
+        :param slots: list of slot names to synchronise.
+        """
         with self._slots_handler.get_local_connection_cursor(dbname=database, options='-c statement_timeout=0') as cur:
             for slot in slots:
                 with self._condition:
@@ -87,6 +114,7 @@ class SlotsAdvanceThread(Thread):
                     self.sync_slot(cur, database, slot, lsn)
 
     def sync_slots(self) -> None:
+        """Synchronise slots for all scheduled databases."""
         with self._condition:
             databases = list(self._scheduled.keys())
         for database in databases:
@@ -99,6 +127,12 @@ class SlotsAdvanceThread(Thread):
                     logger.error('Failed to advance replication slots in database %s: %r', database, e)
 
     def run(self) -> None:
+        """Thread main loop entrypoint.
+
+        .. note::
+            Thread will wait until a sync is scheduled from outside, normally triggered during the HA loop or a wakeup
+            call.
+        """
         while True:
             with self._condition:
                 if not self._scheduled:
@@ -107,6 +141,14 @@ class SlotsAdvanceThread(Thread):
             self.sync_slots()
 
     def schedule(self, advance_slots: Dict[str, Dict[str, int]]) -> Tuple[bool, List[str]]:
+        """Trigger a synchronisation of slots.
+
+        This is the main entrypoint for Patroni HA loop wakeup call.
+
+        :param advance_slots: dictionary containing slots that need to be advanced
+
+        :return: tuple of failure status and a list of slots to be copied
+        """
         with self._condition:
             for database, values in advance_slots.items():
                 self._scheduled[database].update(values)
@@ -118,53 +160,89 @@ class SlotsAdvanceThread(Thread):
         return ret
 
     def on_promote(self) -> None:
+        """Reset state of the daemon."""
         with self._condition:
             self._scheduled.clear()
             self._failed = False
             self._copy_slots = []
 
 
-class SlotsHandler(object):
+class SlotsHandler:
+    """Handler for managing and storing information on replication slots in PostgreSQL.
+
+    :ivar pg_replslot_dir: system location path of the PostgreSQL replication slots.
+    :ivar _logical_slots_processing_queue: yet to be processed logical replication slots on the primary
+    """
 
     def __init__(self, postgresql: 'Postgresql') -> None:
+        """Create an instance with storage attributes for replication slots and schedule the first synchronisation.
+
+        :param postgresql: Calling class instance providing interface to PostgreSQL.
+        """
+        self._force_readiness_check = False
+        self._schedule_load_slots = False
         self._postgresql = postgresql
         self._advance = None
         self._replication_slots: Dict[str, Dict[str, Any]] = {}  # already existing replication slots
-        self._unready_logical_slots: Dict[str, Optional[int]] = {}
+        self._logical_slots_processing_queue: Dict[str, Optional[int]] = {}
         self.pg_replslot_dir = os.path.join(self._postgresql.data_dir, 'pg_replslot')
         self.schedule()
 
-    def _query(self, sql: str, *params: Any) -> Union['cursor', 'Cursor[Any]']:
+    def _query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]:
+        """Helper method for :meth:`Postgresql.query`.
+
+        :param sql: SQL statement to execute.
+        :param params: parameters to pass through to :meth:`Postgresql.query`.
+
+        :returns: query response.
+        """
         return self._postgresql.query(sql, *params, retry=False)
 
     @staticmethod
-    def _copy_items(src: Dict[str, Any], dst: Dict[str, Any], keys: Optional[List[str]] = None) -> None:
+    def _copy_items(src: Dict[str, Any], dst: Dict[str, Any], keys: Optional[Collection[str]] = None) -> None:
+        """Select values from *src* dictionary to update in *dst* dictionary for optional supplied *keys*.
+
+        :param src: source dictionary that *keys* will be looked up from.
+        :param dst: destination dictionary to be updated.
+        :param keys: optional list of keys to be looked up in the source dictionary.
+        """
         dst.update({key: src[key] for key in keys or ('datoid', 'catalog_xmin', 'confirmed_flush_lsn')})
 
     def process_permanent_slots(self, slots: List[Dict[str, Any]]) -> Dict[str, int]:
-        """This methods solves three problems at once (I know, it is weird).
+        """Process replication slot information from the host and prepare information used in subsequent cluster tasks.
+
+        .. note::
+            This methods solves three problems.
+
+            The ``cluster_info_query`` from :class:``Postgresql`` is executed every HA loop and returns information
+            about all replication slots that exists on the current host.
+
+            Based on this information perform the following actions:
+
+            1. For the primary we want to expose to DCS permanent logical slots, therefore build (and return) a dict
+               that maps permanent logical slot names to ``confirmed_flush_lsn``.
+            2. detect if one of the previously known permanent slots is missing and schedule resync.
+            3. Update the local cache with the fresh ``catalog_xmin`` and ``confirmed_flush_lsn`` for every known slot.
 
-        The cluster_info_query from `Postgresql` is executed every HA loop and returns
-        information about all replication slots that exists on the current host.
-        Based on this information we perform the following actions:
-        1. For the primary we want to expose to DCS permanent logical slots, therefore the method
-           builds (and returns) a dict, that maps permanent logical slot names and confirmed_flush_lsns.
-        2. This method also detects if one of the previously known permanent slots got missing and schedules resync.
-        3. Updates the local cache with the fresh catalog_xmin and confirmed_flush_lsn for every known slot.
            This info is used when performing the check of logical slot readiness on standbys.
+
+        :param slots: replication slot information that exists on the current host.
+
+        :return: dictionary of logical slot names to ``confirmed_flush_lsn``.
         """
         ret: Dict[str, int] = {}
 
         slots_dict: Dict[str, Dict[str, Any]] = {slot['slot_name']: slot for slot in slots or []}
-        if slots_dict:
+        for name, value in slots_dict.items():
-            for name, value in slots_dict.items():
+            if name in self._replication_slots:
-                if name in self._replication_slots:
+                if compare_slots(value, self._replication_slots[name], 'datoid'):
-                    if compare_slots(value, self._replication_slots[name], 'datoid'):
+                    if value['type'] == 'logical':
-                        if value['type'] == 'logical':
+                        ret[name] = value['confirmed_flush_lsn']
-                            ret[name] = value['confirmed_flush_lsn']
+                        self._copy_items(value, self._replication_slots[name])
-                            self._copy_items(value, self._replication_slots[name])
                     else:
-                        self._schedule_load_slots = True
+                        self._replication_slots[name]['restart_lsn'] = ret[name] = value['restart_lsn']
+                else:
+                    self._schedule_load_slots = True
 
         # It could happen that the slot was deleted in the background, we want to detect this case
         if any(name not in slots_dict for name in self._replication_slots.keys()):
@@ -173,52 +251,91 @@ class SlotsHandler(object):
         return ret
 
     def load_replication_slots(self) -> None:
+        """Query replication slot information from the database and store it for processing by other tasks.
+
+        .. note::
+            Only supported from PostgreSQL version 9.4 onwards.
+
+        Store replication slot ``name``, ``type``, ``plugin``, ``database`` and ``datoid``.
+        If PostgreSQL version is 10 or newer also store ``catalog_xmin`` and ``confirmed_flush_lsn``.
+
+        When using logical slots, store information separately for slot synchronisation  on replica nodes.
+        """
         if self._postgresql.major_version >= 90400 and self._schedule_load_slots:
             replication_slots: Dict[str, Dict[str, Any]] = {}
-            extra = ", catalog_xmin, pg_catalog.pg_wal_lsn_diff(confirmed_flush_lsn, '0/0')::bigint"\
+            pg_wal_lsn_diff = f"pg_catalog.pg_{self._postgresql.wal_name}_{self._postgresql.lsn_name}_diff"
+            extra = f", catalog_xmin, {pg_wal_lsn_diff}(confirmed_flush_lsn, '0/0')::bigint" \
                 if self._postgresql.major_version >= 100000 else ""
             skip_temp_slots = ' WHERE NOT temporary' if self._postgresql.major_version >= 100000 else ''
-            cursor = self._query('SELECT slot_name, slot_type, plugin, database, datoid'
+            for r in self._query(f"SELECT slot_name, slot_type, {pg_wal_lsn_diff}(restart_lsn, '0/0')::bigint, plugin,"
-                                 '{0} FROM pg_catalog.pg_replication_slots{1}'.format(extra, skip_temp_slots))
+                                 f" database, datoid{extra} FROM pg_catalog.pg_replication_slots{skip_temp_slots}"):
-            for r in cursor:
                 value = {'type': r[1]}
                 if r[1] == 'logical':
-                    value.update(plugin=r[2], database=r[3], datoid=r[4])
+                    value.update(plugin=r[3], database=r[4], datoid=r[5])
                     if self._postgresql.major_version >= 100000:
-                        value.update(catalog_xmin=r[5], confirmed_flush_lsn=r[6])
+                        value.update(catalog_xmin=r[6], confirmed_flush_lsn=r[7])
+                else:
+                    value['restart_lsn'] = r[2]
                 replication_slots[r[0]] = value
             self._replication_slots = replication_slots
             self._schedule_load_slots = False
             if self._force_readiness_check:
-                self._unready_logical_slots = {n: None for n, v in replication_slots.items() if v['type'] == 'logical'}
+                self._logical_slots_processing_queue = {n: None for n, v in replication_slots.items()
+                                                        if v['type'] == 'logical'}
                 self._force_readiness_check = False
 
     def ignore_replication_slot(self, cluster: Cluster, name: str) -> bool:
+        """Check if slot *name* should not be managed by Patroni.
+
+        :param cluster: cluster state information object.
+        :param name: name of the slot to ignore
+
+        :returns: ``True`` if slot *name* matches any slot specified in ``ignore_slots`` configuration,
+                 otherwise will pass through and return result of :meth:`AbstractMPPHandler.ignore_replication_slot`.
+        """
         slot = self._replication_slots[name]
         if cluster.config:
-            for matcher in cluster.config.ignore_slots_matchers:
+            for matcher in global_config.ignore_slots_matchers:
-                if ((matcher.get("name") is None or matcher["name"] == name)
+                if (
-                   and all(not matcher.get(a) or matcher[a] == slot.get(a) for a in ('database', 'plugin', 'type'))):
+                        (matcher.get("name") is None or matcher["name"] == name)
+                        and all(not matcher.get(a) or matcher[a] == slot.get(a)
+                                for a in ('database', 'plugin', 'type'))
+                ):
                     return True
-        return self._postgresql.citus_handler.ignore_replication_slot(slot)
+        return self._postgresql.mpp_handler.ignore_replication_slot(slot)
 
     def drop_replication_slot(self, name: str) -> Tuple[bool, bool]:
-        """Returns a tuple(active, dropped)"""
+        """Drop a named slot from Postgres.
-        cursor = self._query(('WITH slots AS (SELECT slot_name, active'
-                              ' FROM pg_catalog.pg_replication_slots WHERE slot_name = %s),'
-                              ' dropped AS (SELECT pg_catalog.pg_drop_replication_slot(slot_name),'
-                              ' true AS dropped FROM slots WHERE not active) '
-                              'SELECT active, COALESCE(dropped, false) FROM slots'
-                              ' FULL OUTER JOIN dropped ON true'), name)
-        row = cursor.fetchone()
-        if not row:
-            row = (False, False)
-        return row
 
-    def _drop_incorrect_slots(self, cluster: Cluster, slots: Dict[str, Any], paused: bool) -> None:
+        :param name: name of the slot to be dropped.
-        # drop old replication slots which are not presented in desired slots
+
+        :returns: a tuple of ``active`` and ``dropped``. ``active`` is ``True`` if the slot is active,
+                  ``dropped`` is ``True`` if the slot was successfully dropped. If the slot was not found return
+                  ``False`` for both.
+        """
+        rows = self._query(('WITH slots AS (SELECT slot_name, active'
+                            ' FROM pg_catalog.pg_replication_slots WHERE slot_name = %s),'
+                            ' dropped AS (SELECT pg_catalog.pg_drop_replication_slot(slot_name),'
+                            ' true AS dropped FROM slots WHERE not active) '
+                            'SELECT active, COALESCE(dropped, false) FROM slots'
+                            ' FULL OUTER JOIN dropped ON true'), name)
+        return (rows[0][0], rows[0][1]) if rows else (False, False)
+
+    def _drop_incorrect_slots(self, cluster: Cluster, slots: Dict[str, Any]) -> None:
+        """Compare required slots and configured as permanent slots with those found, dropping extraneous ones.
+
+        .. note::
+            Slots that are not contained in *slots* will be dropped.
+            Slots can be filtered out with ``ignore_slots`` configuration.
+
+            Slots that have matching names but do not match attributes in *slots* will also be dropped.
+
+        :param cluster: cluster state information object.
+        :param slots: dictionary of desired slot names as keys with slot attributes as a dictionary value, if known.
+        """
+        # drop old replication slots which are not presented in desired slots.
         for name in set(self._replication_slots) - set(slots):
-            if not paused and not self.ignore_replication_slot(cluster, name):
+            if not global_config.is_paused and not self.ignore_replication_slot(cluster, name):
                 active, dropped = self.drop_replication_slot(name)
                 if dropped:
                     logger.info("Dropped unknown replication slot '%s'", name)
@@ -228,6 +345,8 @@ class SlotsHandler(object):
                         logger.debug("Unable to drop unknown replication slot '%s', slot is still active", name)
                     else:
                         logger.error("Failed to drop replication slot '%s'", name)
+
+        # drop slots with matching names but attributes that do not match, e.g. `plugin` or `database`.
         for name, value in slots.items():
             if name in self._replication_slots and not compare_slots(value, self._replication_slots[name]):
                 logger.info("Trying to drop replication slot '%s' because value is changing from %s to %s",
@@ -239,31 +358,67 @@ class SlotsHandler(object):
                     self._schedule_load_slots = True
 
     def _ensure_physical_slots(self, slots: Dict[str, Any]) -> None:
+        """Create or advance physical replication *slots*.
+
+        Any failures are logged and do not interrupt creation of all *slots*.
+
+        :param slots: A dictionary mapping slot name to slot attributes. This method only considers a slot
+                      if the value is a dictionary with the key ``type`` and a value of ``physical``.
+        """
         immediately_reserve = ', true' if self._postgresql.major_version >= 90600 else ''
         for name, value in slots.items():
-            if name not in self._replication_slots and value['type'] == 'physical':
+            if value['type'] != 'physical':
+                continue
+            if name not in self._replication_slots:
                 try:
-                    self._query(("SELECT pg_catalog.pg_create_physical_replication_slot(%s{0})"
+                    self._query(f"SELECT pg_catalog.pg_create_physical_replication_slot(%s{immediately_reserve})"
-                                 " WHERE NOT EXISTS (SELECT 1 FROM pg_catalog.pg_replication_slots"
+                                f" WHERE NOT EXISTS (SELECT 1 FROM pg_catalog.pg_replication_slots"
-                                 " WHERE slot_type = 'physical' AND slot_name = %s)").format(
+                                f" WHERE slot_type = 'physical' AND slot_name = %s)",
-                                     immediately_reserve), name, name)
+                                name, name)
                 except Exception:
                     logger.exception("Failed to create physical replication slot '%s'", name)
                 self._schedule_load_slots = True
+            elif self._postgresql.can_advance_slots and self._replication_slots[name]['type'] == 'physical':
+                value['restart_lsn'] = self._replication_slots[name]['restart_lsn']
+                lsn = value.get('lsn')
+                if lsn and lsn > value['restart_lsn']:  # The slot has feedback in DCS and needs to be advanced
+                    try:
+                        lsn = format_lsn(lsn)
+                        self._query("SELECT pg_catalog.pg_replication_slot_advance(%s, %s)", name, lsn)
+                    except Exception as exc:
+                        logger.error("Error while advancing replication slot %s to position '%s': %r", name, lsn, exc)
 
     @contextmanager
-    def get_local_connection_cursor(self, **kwargs: Any) -> Generator[Union['cursor', 'Cursor[Any]'], None, None]:
+    def get_local_connection_cursor(self, **kwargs: Any) -> Iterator[Union['cursor', 'Cursor[Any]']]:
-        conn_kwargs = self._postgresql.config.local_connect_kwargs
+        """Create a new database connection to local server.
-        conn_kwargs.update(kwargs)
+
+        Create a non-blocking connection cursor to avoid the situation where an execution of the query of
+        ``pg_replication_slot_advance`` takes longer than the timeout on a HA loop, which could cause a false
+        failure state.
+
+        :param kwargs: Any keyword arguments to pass to :func:`psycopg.connect`.
+
+        :yields: connection cursor object, note implementation varies depending on version of :mod:`psycopg`.
+        """
+        conn_kwargs = {**self._postgresql.connection_pool.conn_kwargs, **kwargs}
         with get_connection_cursor(**conn_kwargs) as cur:
             yield cur
 
     def _ensure_logical_slots_primary(self, slots: Dict[str, Any]) -> None:
+        """Create any missing logical replication *slots* on the primary.
+
+        If the logical slot already exists, copy state information into the replication slots structure stored in the
+        class instance.
+
+        :param slots: Slots that should exist are supplied in a dictionary, mapping slot name to any attributes.
+                      The method will only consider slots that have a value that is a dictionary with a key ``type``
+                      with a value that is ``logical``.
+
+        """
         # Group logical slots to be created by database name
         logical_slots: Dict[str, Dict[str, Dict[str, Any]]] = defaultdict(dict)
         for name, value in slots.items():
             if value['type'] == 'logical':
-                # If the logical already exists, copy some information about it into the original structure
                 if self._replication_slots.get(name, {}).get('datoid'):
                     self._copy_items(self._replication_slots[name], value)
                 else:
@@ -285,54 +440,90 @@ class SlotsHandler(object):
                     self._schedule_load_slots = True
 
     def schedule_advance_slots(self, slots: Dict[str, Dict[str, int]]) -> Tuple[bool, List[str]]:
+        """Wrapper to ensure slots advance daemon thread is started if not already.
+
+        :param slots: dictionary containing slot information.
+
+        :return: tuple with the result of the scheduling of slot advancement: ``failed`` and list of slots to copy.
+        """
         if not self._advance:
             self._advance = SlotsAdvanceThread(self)
         return self._advance.schedule(slots)
 
-    def _ensure_logical_slots_replica(self, cluster: Cluster, slots: Dict[str, Any]) -> List[str]:
+    def _ensure_logical_slots_replica(self, slots: Dict[str, Any]) -> List[str]:
+        """Update logical *slots* on replicas.
+
+        If the logical slot already exists, copy state information into the replication slots structure stored in the
+        class instance. Slots that exist are also advanced if their ``confirmed_flush_lsn`` is greater than the stored
+        state of the slot.
+
+        As logical slots can only be created when the primary is available, pass the list of slots that need to be
+        copied back to the caller. They will be created on replicas with :meth:`SlotsHandler.copy_logical_slots`.
+
+        :param slots: A dictionary mapping slot name to slot attributes. This method only considers a slot
+                      if the value is a dictionary with the key ``type`` and a value of ``logical``.
+
+        :returns: list of slots to be copied from the primary.
+        """
         # Group logical slots to be advanced by database name
         advance_slots: Dict[str, Dict[str, int]] = defaultdict(dict)
-        create_slots: List[str] = []  # And collect logical slots to be created on the replica
+        create_slots: List[str] = []  # Collect logical slots to be created on the replica
+
         for name, value in slots.items():
-            if value['type'] == 'logical':
+            if value['type'] != 'logical':
-                # If the logical already exists, copy some information about it into the original structure
+                continue
-                if self._replication_slots.get(name, {}).get('datoid'):
+
-                    self._copy_items(self._replication_slots[name], value)
+            # If the logical already exists, copy some information about it into the original structure
-                    if cluster.slots and name in cluster.slots:
+            if name in self._replication_slots and compare_slots(value, self._replication_slots[name]):
-                        try:  # Skip slots that doesn't need to be advanced
+                self._copy_items(self._replication_slots[name], value)
-                            if value['confirmed_flush_lsn'] < int(cluster.slots[name]):
+                if 'lsn' in value and value['confirmed_flush_lsn'] < value['lsn']:  # The slot has feedback in DCS
-                                advance_slots[value['database']][name] = int(cluster.slots[name])
+                    # Skip slots that don't need to be advanced
-                        except Exception as e:
+                    advance_slots[value['database']][name] = value['lsn']
-                            logger.error('Failed to parse "%s": %r', cluster.slots[name], e)
+            elif name not in self._replication_slots and 'lsn' in value:
-                elif cluster.slots and name in cluster.slots:  # We want to copy only slots with feedback in a DCS
+                # We want to copy only slots with feedback in a DCS
-                    create_slots.append(name)
+                create_slots.append(name)
+
+        # Slots to be copied from the primary should be removed from the *slots* structure,
+        # otherwise Patroni falsely assumes that they already exist.
+        for name in create_slots:
+            slots.pop(name)
 
         error, copy_slots = self.schedule_advance_slots(advance_slots)
         if error:
             self._schedule_load_slots = True
         return create_slots + copy_slots
 
-    def sync_replication_slots(self, cluster: Cluster, nofailover: bool,
+    def sync_replication_slots(self, cluster: Cluster, tags: Tags) -> List[str]:
-                               replicatefrom: Optional[str] = None, paused: bool = False) -> List[str]:
+        """During the HA loop read, check and alter replication slots found in the cluster.
+
+        Read physical and logical slots from ``pg_replication_slots``, then compare to those configured in the DCS.
+        Drop any slots that do not match those required by configuration and are not configured as permanent.
+        Create any missing physical slots, or advance their position according to feedback stored in DCS.
+        If we are the primary then create logical slots, otherwise if logical slots are known and active create
+        them on replica nodes by copying slot files from the primary.
+
+        :param cluster: object containing stateful information for the cluster.
+        :param tags: reference to an object implementing :class:`Tags` interface.
+
+        :returns: list of logical replication slots names that should be copied from the primary.
+        """
         ret = []
         if self._postgresql.major_version >= 90400 and cluster.config:
             try:
                 self.load_replication_slots()
 
-                slots = cluster.get_replication_slots(self._postgresql.name, self._postgresql.role,
+                slots = cluster.get_replication_slots(self._postgresql, tags, show_error=True)
-                                                      nofailover, self._postgresql.major_version, True)
 
-                self._drop_incorrect_slots(cluster, slots, paused)
+                self._drop_incorrect_slots(cluster, slots)
 
                 self._ensure_physical_slots(slots)
 
-                if self._postgresql.is_leader():
+                if self._postgresql.is_primary():
-                    self._unready_logical_slots.clear()
+                    self._logical_slots_processing_queue.clear()
                     self._ensure_logical_slots_primary(slots)
-                elif cluster.slots and slots:
+                else:
-                    self.check_logical_slots_readiness(cluster, nofailover, replicatefrom)
+                    self.check_logical_slots_readiness(cluster, tags)
-
+                    ret = self._ensure_logical_slots_replica(slots)
-                    ret = self._ensure_logical_slots_replica(cluster, slots)
 
                 self._replication_slots = slots
             except Exception:
@@ -341,61 +532,126 @@ class SlotsHandler(object):
         return ret
 
     @contextmanager
-    def _get_leader_connection_cursor(self, leader: Leader) -> Generator[Union['cursor', 'Cursor[Any]'], None, None]:
+    def _get_leader_connection_cursor(self, leader: Leader) -> Iterator[Union['cursor', 'Cursor[Any]']]:
+        """Create a new database connection to the leader.
+
+        .. note::
+            Uses rewind user credentials because it has enough permissions to read files from PGDATA.
+            Sets the options ``connect_timeout`` to ``3`` and ``statement_timeout`` to ``2000``.
+
+        :param leader: object with information on the leader
+
+        :yields: connection cursor object, note implementation varies depending on version of ``psycopg``.
+        """
         conn_kwargs = leader.conn_kwargs(self._postgresql.config.rewind_credentials)
         conn_kwargs['dbname'] = self._postgresql.database
         with get_connection_cursor(connect_timeout=3, options="-c statement_timeout=2000", **conn_kwargs) as cur:
             yield cur
 
-    def check_logical_slots_readiness(self, cluster: Cluster, nofailover: bool, replicatefrom: Optional[str]) -> None:
+    def check_logical_slots_readiness(self, cluster: Cluster, tags: Tags) -> bool:
+        """Determine whether all known logical slots are synchronised from the leader.
+
+        1) Retrieve the current ``catalog_xmin`` value for the physical slot from the cluster leader, and
+        2) using previously stored list of "unready" logical slots, those which have yet to be checked hence have no
+           stored slot attributes,
+        3) store logical slot ``catalog_xmin`` when the physical slot ``catalog_xmin`` becomes valid.
+
+        :param cluster: object containing stateful information for the cluster.
+        :param tags: reference to an object implementing :class:`Tags` interface.
+
+        :returns: ``False`` if any issue while checking logical slots readiness, ``True`` otherwise.
+        """
         catalog_xmin = None
-        if self._unready_logical_slots and cluster.leader:
+        if self._logical_slots_processing_queue and cluster.leader:
-            slot_name = cluster.get_my_slot_name_on_primary(self._postgresql.name, replicatefrom)
+            slot_name = cluster.get_slot_name_on_primary(self._postgresql.name, tags)
             try:
                 with self._get_leader_connection_cursor(cluster.leader) as cur:
                     cur.execute("SELECT slot_name, catalog_xmin FROM pg_catalog.pg_get_replication_slots()"
                                 " WHERE NOT pg_catalog.pg_is_in_recovery() AND slot_name = ANY(%s)",
-                                ([n for n, v in self._unready_logical_slots.items() if v is None] + [slot_name],))
+                                ([n for n, v in self._logical_slots_processing_queue.items()
+                                  if v is None] + [slot_name],))
                     slots = {row[0]: row[1] for row in cur}
                     if slot_name not in slots:
-                        return logger.warning('Physical slot %s does not exist on the primary', slot_name)
+                        logger.warning('Physical slot %s does not exist on the primary', slot_name)
+                        return False
                     catalog_xmin = slots.pop(slot_name)
             except Exception as e:
-                return logger.error("Failed to check %s physical slot on the primary: %r", slot_name, e)
+                logger.error("Failed to check %s physical slot on the primary: %r", slot_name, e)
-            # Remember catalog_xmin of logical slots on the primary when catalog_xmin of
+                return False
-            # the physical slot became valid. Logical slots on replica will be safe to use after
-            # promote when catalog_xmin of the physical slot overtakes these values.
-            if catalog_xmin is not None:
-                for name, value in slots.items():
-                    self._unready_logical_slots[name] = value
-            else:  # Replica isn't streaming or the hot_standby_feedback isn't enabled
-                try:
-                    cur = self._query("SELECT pg_catalog.current_setting('hot_standby_feedback')::boolean")
-                    row = cur.fetchone()
-                    if row and not row[0]:
-                        logger.error('Logical slot failover requires "hot_standby_feedback".'
-                                     ' Please check postgresql.auto.conf')
-                except Exception as e:
-                    logger.error('Failed to check the hot_standby_feedback setting: %r', e)
-                return  # since `catalog_xmin` isn't valid further checks don't make any sense
 
-        for name in list(self._unready_logical_slots):
+            if not self._update_pending_logical_slot_primary(slots, catalog_xmin):
-            value = self._replication_slots.get(name)
+                return False  # since `catalog_xmin` isn't valid further checks don't make any sense
-            # The logical slot on a replica is safe to use when the physical replica slot on the primary:
+
-            # 1. has a nonzero/non-null catalog_xmin
+        self._ready_logical_slots(catalog_xmin)
-            # 2. has a catalog_xmin that is not newer (greater) than the catalog_xmin of any slot on the standby
+        return True
-            # 3. overtook the catalog_xmin of remembered values of logical slots on the primary.
+
-            if not value or catalog_xmin is not None and\
+    def _update_pending_logical_slot_primary(self, slots: Dict[str, Any], catalog_xmin: Optional[int] = None) -> bool:
-                    self._unready_logical_slots[name] <= catalog_xmin <= value['catalog_xmin']:
+        """Store pending logical slot information for ``catalog_xmin`` on the primary.
-                del self._unready_logical_slots[name]
+
-                if value:
+        Remember ``catalog_xmin`` of logical slots on the primary when ``catalog_xmin`` of the physical slot became
+        valid. Logical slots on replica will be safe to use after promote when ``catalog_xmin`` of the physical slot
+        overtakes these values.
+
+        :param slots: dictionary of slot information from the primary
+        :param catalog_xmin: ``catalog_xmin`` of the physical slot used by this replica to stream changes from primary.
+
+        :returns: ``False`` if any issue was faced while processing, ``True`` otherwise.
+        """
+        if catalog_xmin is not None:
+            for name, value in slots.items():
+                self._logical_slots_processing_queue[name] = value
+            return True
+
+        # Replica isn't streaming or the hot_standby_feedback isn't enabled
+        try:
+            if not self._query("SELECT pg_catalog.current_setting('hot_standby_feedback')::boolean")[0][0]:
+                logger.error('Logical slot failover requires "hot_standby_feedback". Please check postgresql.auto.conf')
+        except Exception as e:
+            logger.error('Failed to check the hot_standby_feedback setting: %r', e)
+        return False
+
+    def _ready_logical_slots(self, primary_physical_catalog_xmin: Optional[int] = None) -> None:
+        """Ready logical slots by comparing primary physical slot ``catalog_xmin`` to logical ``catalog_xmin``.
+
+        The logical slot on a replica is safe to use when the physical replica slot on the primary:
+
+            1. has a nonzero/non-null ``catalog_xmin`` represented by ``primary_physical_xmin``.
+            2. has a ``catalog_xmin`` that is not newer (greater) than the ``catalog_xmin`` of any slot on the standby
+            3. overtook the ``catalog_xmin`` of remembered values of logical slots on the primary.
+
+        :param primary_physical_catalog_xmin: is the value retrieved from ``pg_catalog.pg_get_replication_slots()`` for
+                                              the physical replication slot on the primary.
+        """
+        # Make a copy of processing queue keys as a list as the queue dictionary is modified inside the loop.
+        for name in list(self._logical_slots_processing_queue):
+            primary_logical_catalog_xmin = self._logical_slots_processing_queue[name]
+            standby_logical_slot = self._replication_slots.get(name, {})
+            standby_logical_catalog_xmin = standby_logical_slot.get('catalog_xmin', 0)
+            if TYPE_CHECKING:  # pragma: no cover
+                assert primary_logical_catalog_xmin is not None
+
+            if (
+                    not standby_logical_slot
+                    or primary_physical_catalog_xmin is not None
+                    and primary_logical_catalog_xmin <= primary_physical_catalog_xmin <= standby_logical_catalog_xmin
+            ):
+
+                del self._logical_slots_processing_queue[name]
+
+                if standby_logical_slot:
                     logger.info('Logical slot %s is safe to be used after a failover', name)
 
-    def copy_logical_slots(self, cluster: Cluster, create_slots: List[str]) -> None:
+    def copy_logical_slots(self, cluster: Cluster, tags: Tags, create_slots: List[str]) -> None:
+        """Create logical replication slots on standby nodes.
+
+        :param cluster: object containing stateful information for the cluster.
+        :param tags: reference to an object implementing :class:`Tags` interface.
+        :param create_slots: list of slot names to copy from the primary.
+        """
         leader = cluster.leader
         if not leader:
             return
-        slots = cluster.get_replication_slots(self._postgresql.name, 'replica', False, self._postgresql.major_version)
+        slots = cluster.get_replication_slots(self._postgresql, tags, role='replica')
         copy_slots: Dict[str, Dict[str, Any]] = {}
         with self._get_leader_connection_cursor(leader) as cur:
             try:
@@ -418,34 +674,51 @@ class SlotsHandler(object):
                 logger.error("Failed to copy logical slots from the %s via postgresql connection: %r", leader.name, e)
 
         if copy_slots and self._postgresql.stop():
+            pg_perm.set_permissions_from_data_directory(self._postgresql.data_dir)
             for name, value in copy_slots.items():
-                slot_dir = os.path.join(self._postgresql.slots_handler.pg_replslot_dir, name)
+                slot_dir = os.path.join(self.pg_replslot_dir, name)
                 slot_tmp_dir = slot_dir + '.tmp'
                 if os.path.exists(slot_tmp_dir):
                     shutil.rmtree(slot_tmp_dir)
                 os.makedirs(slot_tmp_dir)
+                os.chmod(slot_tmp_dir, pg_perm.dir_create_mode)
                 fsync_dir(slot_tmp_dir)
-                with open(os.path.join(slot_tmp_dir, 'state'), 'wb') as f:
+                slot_filename = os.path.join(slot_tmp_dir, 'state')
+                with open(slot_filename, 'wb') as f:
+                    os.chmod(slot_filename, pg_perm.file_create_mode)
                     f.write(value['data'])
                     f.flush()
                     os.fsync(f.fileno())
                 if os.path.exists(slot_dir):
                     shutil.rmtree(slot_dir)
                 os.rename(slot_tmp_dir, slot_dir)
+                os.chmod(slot_dir, pg_perm.dir_create_mode)
                 fsync_dir(slot_dir)
-                self._unready_logical_slots[name] = None
+                self._logical_slots_processing_queue[name] = None
-            fsync_dir(self._postgresql.slots_handler.pg_replslot_dir)
+            fsync_dir(self.pg_replslot_dir)
             self._postgresql.start()
 
     def schedule(self, value: Optional[bool] = None) -> None:
+        """Schedule the loading of slot information from the database.
+
+        :param value: the optional value can be used to unschedule if set to ``False`` or force it to be ``True``.
+                      If it is omitted the value will be ``True`` if this PostgreSQL node supports slot replication.
+        """
         if value is None:
             value = self._postgresql.major_version >= 90400
         self._schedule_load_slots = self._force_readiness_check = value
 
     def on_promote(self) -> None:
+        """Entry point from HA cycle used when a standby node is to be promoted to primary.
+
+        .. note::
+            If logical replication slot synchronisation is enabled then slot advancement will be triggered.
+            If any logical slots that were copied are yet to be confirmed as ready a warning message will be logged.
+
+        """
         if self._advance:
             self._advance.on_promote()
 
-        if self._unready_logical_slots:
+        if self._logical_slots_processing_queue:
             logger.warning('Logical replication slots that might be unsafe to use after promote: %s',
-                           set(self._unready_logical_slots))
+                           set(self._logical_slots_processing_queue))

patroni/postgresql/sync.py

@@ -3,8 +3,9 @@ import re
 import time
 
 from copy import deepcopy
-from typing import Collection, List, NamedTuple, Tuple, TYPE_CHECKING
+from typing import Collection, List, NamedTuple, Optional, TYPE_CHECKING
 
+from .. import global_config
 from ..collections import CaseInsensitiveDict, CaseInsensitiveSet
 from ..dcs import Cluster
 from ..psycopg import quote_ident as _quote_ident
@@ -137,7 +138,7 @@ def parse_sync_standby_names(value: str) -> _SSN:
             if len(synclist) == i + 1:  # except the last token
                 raise ValueError("Unparseable synchronous_standby_names value %r: Unexpected token %s %r at %d" %
                                  (value, a_type, a_value, a_pos))
-            elif a_type != 'comma':
+            if a_type != 'comma':
                 raise ValueError("Unparseable synchronous_standby_names value %r: ""Got token %s %r while"
                                  " expecting comma at %d" % (value, a_type, a_value, a_pos))
         elif a_type in {'ident', 'first', 'any'}:
@@ -153,6 +154,94 @@ def parse_sync_standby_names(value: str) -> _SSN:
     return _SSN(sync_type, has_star, num, members)
 
 
+class _SyncState(NamedTuple):
+    """Class representing the current synchronous state.
+
+    :ivar sync_type: possible values: ``off``, ``priority``, ``quorum``
+    :ivar numsync: how many nodes are required to be synchronous (according to ``synchronous_standby_names``).
+                   Is ``0`` if ``synchronous_standby_names`` value is invalid or contains ``*``.
+    :ivar numsync_confirmed: how many nodes are known to be synchronous according to the ``pg_stat_replication`` view.
+                             Only nodes that caught up with the :attr:`SyncHandler._primary_flush_lsn` are counted.
+    :ivar sync: collection of synchronous node names. In case of quorum commit all nodes listed
+                in ``synchronous_standby_names``, otherwise nodes that are confirmed to be synchronous according
+                to the ``pg_stat_replication`` view.
+    :ivar active: collection of node names that are streaming and have no restrictions to become synchronous.
+    """
+    sync_type: str
+    numsync: int
+    numsync_confirmed: int
+    sync: CaseInsensitiveSet
+    active: CaseInsensitiveSet
+
+
+class _Replica(NamedTuple):
+    """Class representing a single replica that is eligible to be synchronous.
+
+    Attributes are taken from ``pg_stat_replication`` view and respective ``Cluster.members``.
+
+    :ivar pid: PID of walsender process.
+    :ivar application_name: matches with the ``Member.name``.
+    :ivar sync_state: possible values are: ``async``, ``potential``, ``quorum``, and ``sync``.
+    :ivar lsn: ``write_lsn``, ``flush_lsn``, or ``replay_lsn``, depending on the value of ``synchronous_commit`` GUC.
+    :ivar nofailover: whether the corresponding member has ``nofailover`` tag set to ``True``.
+    """
+    pid: int
+    application_name: str
+    sync_state: str
+    lsn: int
+    nofailover: bool
+
+
+class _ReplicaList(List[_Replica]):
+    """A collection of :class:``_Replica`` objects.
+
+    Values are reverse ordered by ``_Replica.sync_state`` and ``_Replica.lsn``.
+    That is, first there will be replicas that have ``sync_state`` == ``sync``, even if they are not
+    the most up-to-date in term of write/flush/replay LSN. It helps to keep the result of chosing new
+    synchronous nodes consistent in case if a synchronous standby member is slowed down OR async node
+    is receiving changes faster than the sync member. Such cases would trigger sync standby member
+    swapping, but only if lag on this member is exceeding a threshold (``maximum_lag_on_syncnode``).
+
+    :ivar max_lsn: maximum value of ``_Replica.lsn`` among all values. In case if there is just one
+                   element in the list we take value of ``pg_current_wal_flush_lsn()``.
+    """
+
+    def __init__(self, postgresql: 'Postgresql', cluster: Cluster) -> None:
+        """Create :class:``_ReplicaList`` object.
+
+        :param postgresql: reference to :class:``Postgresql`` object.
+        :param cluster: currently known cluster state from DCS.
+        """
+        super().__init__()
+
+        # We want to prioritize candidates based on `write_lsn``, ``flush_lsn``, or ``replay_lsn``.
+        # Which column exactly to pick depends on the values of ``synchronous_commit`` GUC.
+        sort_col = {
+            'remote_apply': 'replay',
+            'remote_write': 'write'
+        }.get(postgresql.synchronous_commit(), 'flush') + '_lsn'
+
+        members = CaseInsensitiveDict({m.name: m for m in cluster.members})
+        for row in postgresql.pg_stat_replication():
+            member = members.get(row['application_name'])
+
+            # We want to consider only rows from ``pg_stat_replication` that:
+            # 1. are known to be streaming (write/flush/replay LSN are not NULL).
+            # 2. can be mapped to a ``Member`` of the ``Cluster``:
+            #   a. ``Member`` doesn't have ``nosync`` tag set;
+            #   b. PostgreSQL on the member is known to be running and accepting client connections.
+            if member and row[sort_col] is not None and member.is_running and not member.nosync:
+                self.append(_Replica(row['pid'], row['application_name'],
+                                     row['sync_state'], row[sort_col], bool(member.nofailover)))
+
+        # Prefer replicas that are in state ``sync`` and with higher values of ``write``/``flush``/``replay`` LSN.
+        self.sort(key=lambda r: (r.sync_state, r.lsn), reverse=True)
+
+        # When checking ``maximum_lag_on_syncnode`` we want to compare with the most
+        # up-to-date replica otherwise with cluster LSN if there is only one replica.
+        self.max_lsn = max(self, key=lambda x: x.lsn).lsn if len(self) > 1 else postgresql.last_operation()
+
+
 class SyncHandler(object):
     """Class responsible for working with the `synchronous_standby_names`.
 
@@ -201,92 +290,121 @@ BEGIN
 END;$$""")
         self._postgresql.reset_cluster_info_state(None)  # Reset internal cache to query fresh values
 
-    def current_state(self, cluster: Cluster) -> Tuple[CaseInsensitiveSet, CaseInsensitiveSet]:
+    def _process_replica_readiness(self, cluster: Cluster, replica_list: _ReplicaList) -> None:
-        """Finds best candidates to be the synchronous standbys.
+        """Flags replicas as truly "synchronous" when they have caught up with ``_primary_flush_lsn``.
+
+        :param cluster: current cluster topology from DCS
+        :param replica_list: collection of replicas that we want to evaluate.
+        """
+        for replica in replica_list:
+            # if standby name is listed in the /sync key we can count it as synchronous, otherwise
+            # it becomes really synchronous when sync_state = 'sync' and it is known that it managed to catch up
+            if replica.application_name not in self._ready_replicas\
+                    and replica.application_name in self._ssn_data.members:
+                if global_config.is_quorum_commit_mode:
+                    # When quorum commit is enabled we can't check against cluster.sync because nodes
+                    # are written there when at least one of them caught up with _primary_flush_lsn.
+                    if replica.lsn >= self._primary_flush_lsn\
+                            and (replica.sync_state == 'quorum'
+                                 or (not self._postgresql.supports_quorum_commit
+                                     and replica.sync_state in ('sync', 'potential'))):
+                        self._ready_replicas[replica.application_name] = replica.pid
+                elif cluster.sync.matches(replica.application_name)\
+                        or replica.sync_state == 'sync' and replica.lsn >= self._primary_flush_lsn:
+                    # if standby name is listed in the /sync key we can count it as synchronous, otherwise it becomes
+                    # "really" synchronous when sync_state = 'sync' and we known that it managed to catch up
+                    self._ready_replicas[replica.application_name] = replica.pid
+
+    def current_state(self, cluster: Cluster) -> _SyncState:
+        """Find the best candidates to be the synchronous standbys.
 
         Current synchronous standby is always preferred, unless it has disconnected or does not want to be a
         synchronous standby any longer.
 
         Standbys are selected based on values from the global configuration:
-        - `maximum_lag_on_syncnode`: would help swapping unhealthy sync replica in case if it stops
-          responding (or hung). Please set the value high enough so it won't unncessarily swap sync
-          standbys during high loads. Any value less or equal of 0 keeps the behavior backward compatible.
-          Please note that it will not also swap sync standbys in case where all replicas are hung.
-        - `synchronous_node_count`: controlls how many nodes should be set as synchronous.
 
-        :returns: tuple of candidates :class:`CaseInsensitiveSet` and synchronous standbys :class:`CaseInsensitiveSet`.
+            - ``maximum_lag_on_syncnode``: would help swapping unhealthy sync replica in case it stops
+              responding (or hung). Please set the value high enough, so it won't unnecessarily swap sync
+              standbys during high loads. Any value less or equal to ``0`` keeps the behavior backwards compatible.
+              Please note that it will also not swap sync standbys when all replicas are hung.
+
+            - ``synchronous_node_count``: controls how many nodes should be set as synchronous.
+
+        :param cluster: current cluster topology from DCS
+
+        :returns: current synchronous replication state as a :class:`_SyncState` object
         """
         self._handle_synchronous_standby_names_change()
 
-        # Pick candidates based on who has higher replay/remote_write/flush lsn.
+        replica_list = _ReplicaList(self._postgresql, cluster)
-        sort_col = {
+        self._process_replica_readiness(cluster, replica_list)
-            'remote_apply': 'replay',
-            'remote_write': 'write'
-        }.get(self._postgresql.synchronous_commit(), 'flush') + '_lsn'
 
-        pg_stat_replication = [(r['pid'], r['application_name'], r['sync_state'], r[sort_col])
+        active = CaseInsensitiveSet()
-                               for r in self._postgresql.pg_stat_replication()
-                               if r[sort_col] is not None]
-
-        members = CaseInsensitiveDict({m.name: m for m in cluster.members})
-        replica_list: List[Tuple[int, str, str, int, bool]] = []
-        # pg_stat_replication.sync_state has 4 possible states - async, potential, quorum, sync.
-        # That is, alphabetically they are in the reversed order of priority.
-        # Since we are doing reversed sort on (sync_state, lsn) tuples, it helps to keep the result
-        # consistent in case if a synchronous standby member is slowed down OR async node receiving
-        # changes faster than the sync member (very rare but possible).
-        # Such cases would trigger sync standby member swapping, but only if lag on a sync node exceeding a threshold.
-        for pid, app_name, sync_state, replica_lsn in sorted(pg_stat_replication, key=lambda r: r[2:4], reverse=True):
-            member = members.get(app_name)
-            if member and member.is_running and not member.tags.get('nosync', False):
-                replica_list.append((pid, member.name, sync_state, replica_lsn, bool(member.nofailover)))
-
-        max_lsn = max(replica_list, key=lambda x: x[3])[3]\
-            if len(replica_list) > 1 else self._postgresql.last_operation()
-
-        if TYPE_CHECKING:  # pragma: no cover
-            assert self._postgresql.global_config is not None
-        sync_node_count = self._postgresql.global_config.synchronous_node_count\
-            if self._postgresql.supports_multiple_sync else 1
-        sync_node_maxlag = self._postgresql.global_config.maximum_lag_on_syncnode
-
-        candidates = CaseInsensitiveSet()
         sync_nodes = CaseInsensitiveSet()
+        numsync_confirmed = 0
+
+        sync_node_count = global_config.synchronous_node_count if self._postgresql.supports_multiple_sync else 1
+        sync_node_maxlag = global_config.maximum_lag_on_syncnode
+
         # Prefer members without nofailover tag. We are relying on the fact that sorts are guaranteed to be stable.
-        for pid, app_name, sync_state, replica_lsn, _ in sorted(replica_list, key=lambda x: x[4]):
+        for replica in sorted(replica_list, key=lambda x: x.nofailover):
-            # if standby name is listed in the /sync key we can count it as synchronous, otherwice
+            if sync_node_maxlag <= 0 or replica_list.max_lsn - replica.lsn <= sync_node_maxlag:
-            # it becomes really synchronous when sync_state = 'sync' and it is known that it managed to catch up
+                if global_config.is_quorum_commit_mode:
-            if app_name not in self._ready_replicas and app_name in self._ssn_data.members and\
+                    # We do not add nodes with `nofailover` enabled because that reduces availability.
-                    (cluster.sync.matches(app_name) or sync_state == 'sync' and replica_lsn >= self._primary_flush_lsn):
+                    # We need to check LSN quorum only among nodes that are promotable because
-                self._ready_replicas[app_name] = pid
+                    # there is a chance that a non-promotable node is ahead of a promotable one.
+                    if not replica.nofailover or len(active) < sync_node_count:
+                        if replica.application_name in self._ready_replicas:
+                            numsync_confirmed += 1
+                        active.add(replica.application_name)
+                else:
+                    active.add(replica.application_name)
+                    if replica.sync_state == 'sync' and replica.application_name in self._ready_replicas:
+                        sync_nodes.add(replica.application_name)
+                        numsync_confirmed += 1
+                    if len(active) >= sync_node_count:
+                        break
 
-            if sync_node_maxlag <= 0 or max_lsn - replica_lsn <= sync_node_maxlag:
+        if global_config.is_quorum_commit_mode:
-                candidates.add(app_name)
+            sync_nodes = CaseInsensitiveSet() if self._ssn_data.has_star else self._ssn_data.members
-                if sync_state == 'sync' and app_name in self._ready_replicas:
-                    sync_nodes.add(app_name)
-            if len(candidates) >= sync_node_count:
-                break
 
-        return candidates, sync_nodes
+        return _SyncState(
+            self._ssn_data.sync_type,
+            0 if self._ssn_data.has_star else self._ssn_data.num,
+            numsync_confirmed,
+            sync_nodes,
+            active)
 
-    def set_synchronous_standby_names(self, sync: Collection[str]) -> None:
+    def set_synchronous_standby_names(self, sync: Collection[str], num: Optional[int] = None) -> None:
-        """Constructs and sets "synchronous_standby_names" GUC value.
+        """Constructs and sets ``synchronous_standby_names`` GUC value.
+
+        .. note::
+            standbys in ``synchronous_standby_names`` will be sorted by name.
 
         :param sync: set of nodes to sync to
+        :param num: specifies number of nodes to sync to. The *num* is set only in case if quorum commit is enabled
         """
-        has_asterisk = '*' in sync
+        # Special case. If sync nodes set is empty but requested num of sync nodes >= 1
+        # we want to set synchronous_standby_names to '*'
+        has_asterisk = '*' in sync or num and num >= 1 and not sync
         if has_asterisk:
             sync = ['*']
         else:
-            sync = [quote_ident(x) for x in sync]
+            sync = [quote_ident(x) for x in sorted(sync)]
 
         if self._postgresql.supports_multiple_sync and len(sync) > 1:
-            sync_param = '{0} ({1})'.format(len(sync), ','.join(sync))
+            if num is None:
+                num = len(sync)
+            sync_param = ','.join(sync)
         else:
             sync_param = next(iter(sync), None)
 
+        if global_config.is_quorum_commit_mode and sync or self._postgresql.supports_multiple_sync and len(sync) > 1:
+            prefix = 'ANY ' if global_config.is_quorum_commit_mode and self._postgresql.supports_quorum_commit else ''
+            sync_param = f'{prefix}{num} ({sync_param})'
+
         if not (self._postgresql.config.set_synchronous_standby_names(sync_param)
-                and self._postgresql.state == 'running' and self._postgresql.is_leader()) or has_asterisk:
+                and self._postgresql.state == 'running' and self._postgresql.is_primary()) or has_asterisk:
             return
 
         time.sleep(0.1)  # Usualy it takes 1ms to reload postgresql.conf, but we will give it 100ms

patroni/postgresql/validator.py

@@ -1,11 +1,11 @@
 import abc
 from copy import deepcopy
 import logging
-import os
 import yaml
 
 from typing import Any, Dict, Iterator, List, MutableMapping, Optional, Tuple, Type, Union
 
+from .available_parameters import get_validator_files, PathLikeObj
 from ..collections import CaseInsensitiveDict, CaseInsensitiveSet
 from ..exceptions import PatroniException
 from ..utils import parse_bool, parse_int, parse_real
@@ -178,10 +178,11 @@ class ValidatorFactory:
 
         :returns: the Patroni validator object that corresponds to the specification found in *validator*.
 
-        :raises :class:`ValidatorFactoryNoType`: if *validator* contains no ``type`` key.
+        :raises:
-        :raises :class:`ValidatorFactoryInvalidType`: if ``type`` key from *validator* contains an invalid value.
+            :class:`ValidatorFactoryNoType`: if *validator* contains no ``type`` key.
-        :raises :class:`ValidatorFactoryInvalidSpec`: if *validator* contains an invalid set of attributes for the
+            :class:`ValidatorFactoryInvalidType`: if ``type`` key from *validator* contains an invalid value.
-            given ``type``.
+            :class:`ValidatorFactoryInvalidSpec`: if *validator* contains an invalid set of attributes for the given
+                ``type``.
 
         :Example:
 
@@ -257,18 +258,19 @@ class InvalidGucValidatorsFile(PatroniException):
     """Raised when reading or parsing of a YAML file faces an issue."""
 
 
-def _read_postgres_gucs_validators_file(file: str) -> Dict[str, Any]:
+def _read_postgres_gucs_validators_file(file: PathLikeObj) -> Dict[str, Any]:
     """Read an YAML file and return the corresponding Python object.
 
-    :param file: path to the file to be read. It is expected to be encoded with ``UTF-8``, and to be a YAML document.
+    :param file: path-like object to read from. It is expected to be encoded with ``UTF-8``, and to be a YAML document.
 
     :returns: the YAML content parsed into a Python object. If any issue is faced while reading/parsing the file, then
         return ``None``.
 
-    :raises :class:`InvalidGucValidatorsFile`: if faces an issue while reading or parsing *file*.
+    :raises:
+        :class:`InvalidGucValidatorsFile`: if faces an issue while reading or parsing *file*.
     """
     try:
-        with open(file, encoding='UTF-8') as stream:
+        with file.open(encoding='UTF-8') as stream:
             return yaml.safe_load(stream)
     except Exception as exc:
         raise InvalidGucValidatorsFile(
@@ -288,7 +290,7 @@ def _load_postgres_gucs_validators() -> None:
     Any problem faced while reading or parsing files will be logged as a ``WARNING`` by the child function, and the
     corresponding file or validator will be ignored.
 
-    By default Patroni only ships the file ``0_postgres.yml``, which contains Community Postgres GUCs validators, but
+    By default, Patroni only ships the file ``0_postgres.yml``, which contains Community Postgres GUCs validators, but
     that behavior can be extended. For example: if a vendor wants to add GUC validators to Patroni for covering a custom
     Postgres build, then they can create their custom YAML files under ``available_parameters`` directory.
 
@@ -298,8 +300,10 @@ def _load_postgres_gucs_validators() -> None:
             writes them to ``postgresql.conf`` if running PG 12 and above).
 
     Then, each of these sections, if specified, may contain one or more attributes with the following structure:
+
         * key: the name of a GUC;
         * value: a list of validators. Each item in the list must contain a ``type`` attribute, which must be one among:
+
             * ``Bool``; or
             * ``Integer``; or
             * ``Real``; or
@@ -311,6 +315,7 @@ def _load_postgres_gucs_validators() -> None:
             class in this module.
 
     .. seealso::
+
         * :class:`Bool`;
         * :class:`Integer`;
         * :class:`Real`;
@@ -323,77 +328,64 @@ def _load_postgres_gucs_validators() -> None:
         This is a sample content for an YAML file based on Postgres GUCs, showing each of the supported types and
         sections:
 
-        ```yaml
+            .. code-block:: yaml
-        parameters:
+
-          archive_command:
+                parameters:
-          - type: String
+                  archive_command:
-            version_from: 90300
+                  - type: String
-            version_till: null
+                    version_from: 90300
-          archive_mode:
+                    version_till: null
-          - type: Bool
+                  archive_mode:
-            version_from: 90300
+                  - type: Bool
-            version_till: 90500
+                    version_from: 90300
-          - type: EnumBool
+                    version_till: 90500
-            version_from: 90500
+                  - type: EnumBool
-            version_till: null
+                    version_from: 90500
-            possible_values:
+                    version_till: null
-            - always
+                    possible_values:
-          archive_timeout:
+                    - always
-          - type: Integer
+                  archive_timeout:
-            version_from: 90300
+                  - type: Integer
-            version_till: null
+                    version_from: 90300
-            min_val: 0
+                    version_till: null
-            max_val: 1073741823
+                    min_val: 0
-            unit: s
+                    max_val: 1073741823
-          autovacuum_vacuum_cost_delay:
+                    unit: s
-          - type: Integer
+                  autovacuum_vacuum_cost_delay:
-            version_from: 90300
+                  - type: Integer
-            version_till: 120000
+                    version_from: 90300
-            min_val: -1
+                    version_till: 120000
-            max_val: 100
+                    min_val: -1
-            unit: ms
+                    max_val: 100
-          - type: Real
+                    unit: ms
-            version_from: 120000
+                  - type: Real
-            version_till: null
+                    version_from: 120000
-            min_val: -1
+                    version_till: null
-            max_val: 100
+                    min_val: -1
-            unit: ms
+                    max_val: 100
-          client_min_messages:
+                    unit: ms
-          - type: Enum
+                  client_min_messages:
-            version_from: 90300
+                  - type: Enum
-            version_till: null
+                    version_from: 90300
-            possible_values:
+                    version_till: null
-            - debug5
+                    possible_values:
-            - debug4
+                    - debug5
-            - debug3
+                    - debug4
-            - debug2
+                    - debug3
-            - debug1
+                    - debug2
-            - log
+                    - debug1
-            - notice
+                    - log
-            - warning
+                    - notice
-            - error
+                    - warning
-        recovery_parameters:
+                    - error
-          archive_cleanup_command:
+                recovery_parameters:
-          - type: String
+                  archive_cleanup_command:
-            version_from: 90300
+                  - type: String
-            version_till: null
+                    version_from: 90300
-        ```
+                    version_till: null
+
     """
-    conf_dir = os.path.join(
+    for file in get_validator_files():
-        os.path.dirname(os.path.abspath(__file__)),
-        'available_parameters',
-    )
-    yaml_files: List[str] = []
-
-    for root, _, files in os.walk(conf_dir):
-        for file in sorted(files):
-            full_path = os.path.join(root, file)
-            if file.lower().endswith(('.yml', '.yaml')):
-                yaml_files.append(full_path)
-            else:
-                logger.info('Ignored a non-YAML file found under `available_parameters` directory: `%s`.', full_path)
-
-    for file in yaml_files:
         try:
             config: Dict[str, Any] = _read_postgres_gucs_validators_file(file)
         except InvalidGucValidatorsFile as exc:
@@ -432,13 +424,15 @@ def _transform_parameter_value(validators: MutableMapping[str, Tuple[_Transforma
     :param value: value of the Postgres GUC.
     :param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres
         GUC. Used for a couple purposes:
+
         * Disallow writing GUCs to ``postgresql.conf`` (or ``recovery.conf``) that does not exist in Postgres *version*;
         * Avoid ignoring GUC *name* if it does not have a validator in *validators*, but is a valid GUC in Postgres
-            *version*.
+          *version*.
 
     :returns: the return value may be one among:
-        * *value* transformed to the expected format for GUC *name* in Postgres *version*, if *name* is present in
+
-            *available_gucs* and has a validator in *validators* for the corresponding Postgres *version*; or
+        * *value* transformed to the expected format for GUC *name* in Postgres *version*, if *name* is present
+          in *available_gucs* and has a validator in *validators* for the corresponding Postgres *version*; or
         * The own *value* if *name* is present in *available_gucs* but not in *validators*; or
         * ``None`` if *name* is not present in *available_gucs*.
     """
@@ -462,11 +456,13 @@ def transform_postgresql_parameter_value(version: int, name: str, value: Any,
     :param value: value of the Postgres GUC.
     :param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres
         GUC. Used for a couple purposes:
-        * Disallow writing GUCs to ``postgresql.conf`` that does not exist in Postgres *version*;
-        * Avoid ignoring GUC *name* if it does not have a validator in ``parameters``, but is a valid GUC in Postgres
-            *version*.
 
-    :returns: The return value may be one among
+            * Disallow writing GUCs to ``postgresql.conf`` that does not exist in Postgres *version*;
+            * Avoid ignoring GUC *name* if it does not have a validator in ``parameters``, but is a valid GUC in
+                Postgres *version*.
+
+    :returns: The return value may be one among:
+
         * The original *value* if *name* seems to be an extension GUC (contains a period '.'); or
         * ``None`` if **name** is a recovery GUC; or
         * *value* transformed to the expected format for GUC *name* in Postgres *version* using validators defined in
@@ -490,10 +486,11 @@ def transform_recovery_parameter_value(version: int, name: str, value: Any,
     :param value: value of the Postgres recovery GUC.
     :param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres
         GUC. Used for a couple purposes:
-        * Disallow writing GUCs to ``recovery.conf`` (or ``postgresql.conf`` depending on *version*), that does not
+
-        exist in Postgres *version*;
+            * Disallow writing GUCs to ``recovery.conf`` (or ``postgresql.conf`` depending on *version*), that does not
-        * Avoid ignoring recovery GUC *name* if it does not have a validator in ``recovery_parameters``, but is a valid
+                exist in Postgres *version*;
-            GUC in Postgres *version*.
+            * Avoid ignoring recovery GUC *name* if it does not have a validator in ``recovery_parameters``, but is a
+                valid GUC in Postgres *version*.
 
     :returns: *value* transformed to the expected format for recovery GUC *name* in Postgres *version* using validators
         defined in ``recovery_parameters``. It can also return ``None``. See :func:`_transform_parameter_value`.

patroni/psycopg.py

@@ -1,7 +1,8 @@
-"""Abstraction layer for ``psycopg`` module.
+"""Abstraction layer for :mod:`psycopg` module.
 
-This module is able to handle both ``pyscopg2`` and ``psycopg3``, and it exposes a common interface for both.
+This module is able to handle both :mod:`pyscopg2` and :mod:`psycopg`, and it exposes a common interface for both.
-``psycopg2`` takes precedence. ``psycopg3`` will only be used if ``psycopg2`` is either absent or older than ``2.5.4``.
+:mod:`psycopg2` takes precedence. :mod:`psycopg` will only be used if :mod:`psycopg2` is either absent or older than
+``2.5.4``.
 """
 from typing import Any, Optional, TYPE_CHECKING, Union
 if TYPE_CHECKING:  # pragma: no cover
@@ -28,7 +29,7 @@ try:
         """Quote *value* as a SQL literal.
 
         .. note::
-            *value* is quoted through ``psycopg`` adapters.
+            *value* is quoted through :mod:`psycopg2` adapters.
 
         :param value: value to be quoted.
         :param conn: if a connection is given then :func:`quote_literal` checks if any special handling based on server
@@ -44,14 +45,14 @@ except ImportError:
     from psycopg import connect as __connect, sql, Error, DatabaseError, OperationalError, ProgrammingError
 
     def _connect(dsn: Optional[str] = None, **kwargs: Any) -> 'Connection[Any]':
-        """Call ``psycopg.connect`` with ``dsn`` and ``**kwargs``.
+        """Call :func:`psycopg.connect` with *dsn* and ``**kwargs``.
 
         .. note::
             Will create ``server_version`` attribute in the returning connection, so it keeps compatibility with the
-            object that would be returned by ``psycopg2.connect``.
+            object that would be returned by :func:`psycopg2.connect`.
 
-        :param dsn: DSN to call ``psycopg.connect`` with.
+        :param dsn: DSN to call :func:`psycopg.connect` with.
-        :param kwargs: keyword arguments to call ``psycopg.connect`` with.
+        :param kwargs: keyword arguments to call :func:`psycopg.connect` with.
 
         :returns: a connection to the database.
         """
@@ -89,11 +90,11 @@ def connect(*args: Any, **kwargs: Any) -> Union['connection', 'Connection[Any]']
         It also enforces ``search_path=pg_catalog`` for non-replication connections to mitigate security issues as
         Patroni relies on superuser connections.
 
-    :param args: positional arguments to call ``connect`` function from ``psycopg`` module.
+    :param args: positional arguments to call :func:`~psycopg.connect` function from :mod:`psycopg` module.
-    :param kwargs: keyword arguments to call ``connect`` function from ``psycopg`` module.
+    :param kwargs: keyword arguments to call :func:`~psycopg.connect` function from :mod:`psycopg` module.
 
-    :returns: a connection to the database. Can be either a :class:`psycopg.Connection` if using ``psycopg3``, or a
+    :returns: a connection to the database. Can be either a :class:`psycopg.Connection` if using :mod:`psycopg`, or a
-        :class:`psycopg2.extensions.connection` if using ``psycopg2``.
+        :class:`psycopg2.extensions.connection` if using :mod:`psycopg2`.
     """
     if kwargs and 'replication' not in kwargs and kwargs.get('fallback_application_name') != 'Patroni ctl':
         options = [kwargs['options']] if 'options' in kwargs else []
@@ -109,7 +110,7 @@ def quote_ident(value: Any, conn: Optional[Union['cursor', 'connection', 'Connec
 
     :param value: value to be quoted.
     :param conn: connection to evaluate the returning string into. Can be either a :class:`psycopg.Connection` if
-        using ``psycopg3``, or a :class:`psycopg2.extensions.connection` if using ``psycopg2``.
+        using :mod:`psycopg`, or a :class:`psycopg2.extensions.connection` if using :mod:`psycopg2`.
 
     :returns: *value* quoted as a SQL identifier.
     """

patroni/quorum.py

@@ -0,0 +1,431 @@
+"""Implement state machine to manage ``synchronous_standby_names`` GUC and ``/sync`` key in DCS."""
+import logging
+
+from typing import Collection, Iterator, NamedTuple, Optional
+
+from .collections import CaseInsensitiveSet
+from .exceptions import PatroniException
+
+logger = logging.getLogger(__name__)
+
+
+class Transition(NamedTuple):
+    """Object describing transition of ``/sync`` or ``synchronous_standby_names`` to the new state.
+
+    .. note::
+        Object attributes represent the new state.
+
+    :ivar transition_type: possible values:
+
+        * ``sync`` - indicates that we needed to update ``synchronous_standby_names``.
+        * ``quorum`` - indicates that we need to update ``/sync`` key in DCS.
+        * ``restart`` - caller should stop iterating over transitions and restart :class:`QuorumStateResolver`.
+    :ivar leader: the new value of the ``leader`` field in the ``/sync`` key.
+    :ivar num: the new value of the synchronous nodes count in ``synchronous_standby_names`` or value of the ``quorum``
+               field in the ``/sync`` key for :attr:`transition_type` values ``sync`` and ``quorum`` respectively.
+    :ivar names: the new value of node names listed in ``synchronous_standby_names`` or value of ``voters``
+                 field in the ``/sync`` key  for :attr:`transition_type` values ``sync`` and ``quorum`` respectively.
+    """
+
+    transition_type: str
+    leader: str
+    num: int
+    names: CaseInsensitiveSet
+
+
+class QuorumError(PatroniException):
+    """Exception indicating that the quorum state is broken."""
+
+
+class QuorumStateResolver:
+    """Calculates a list of state transitions and yields them as :class:`Transition` named tuples.
+
+    Synchronous replication state is set in two places:
+
+    * PostgreSQL configuration sets how many and which nodes are needed for a commit to succeed, abbreviated as
+      ``numsync`` and ``sync`` set here;
+    * DCS contains information about how many and which nodes need to be interrogated to be sure to see an wal position
+      containing latest confirmed commit, abbreviated as ``quorum`` and ``voters`` set.
+
+    .. note::
+        Both of above pairs have the meaning "ANY n OF set".
+
+        The number of nodes needed for commit to succeed, ``numsync``, is also called the replication factor.
+
+    To guarantee zero transaction loss on failover we need to keep the invariant that at all times any subset of
+    nodes that can acknowledge a commit overlaps with any subset of nodes that can achieve quorum to promote a new
+    leader. Given a desired replication factor and a set of nodes able to participate in sync replication there
+    is one optimal state satisfying this condition. Given the node set ``active``, the optimal state is::
+
+        sync = voters = active
+
+        numsync = min(sync_wanted, len(active))
+
+        quorum = len(active) - numsync
+
+    We need to be able to produce a series of state changes that take the system to this desired state from any
+    other arbitrary state given arbitrary changes is node availability, configuration and interrupted transitions.
+
+    To keep the invariant the rule to follow is that when increasing ``numsync`` or ``quorum``, we need to perform the
+    increasing operation first. When decreasing either, the decreasing operation needs to be performed later. In other
+    words:
+
+    * If a user increases ``synchronous_node_count`` configuration, first we increase ``synchronous_standby_names``
+      (``numsync``), then we decrease ``quorum`` field in the ``/sync`` key;
+    * If a user decreases ``synchronous_node_count`` configuration, first we increase ``quorum`` field in the ``/sync``
+      key, then we decrease ``synchronous_standby_names`` (``numsync``).
+
+    Order of adding or removing nodes from ``sync`` and ``voters`` depends on the state of
+    ``synchronous_standby_names``.
+
+    When adding new nodes::
+
+        if ``sync`` (``synchronous_standby_names``) is empty:
+            add new nodes first to ``sync`` and then to ``voters`` when ``numsync_confirmed`` > ``0``.
+        else:
+            add new nodes first to ``voters`` and then to ``sync``.
+
+    When removing nodes::
+
+        if ``sync`` (``synchronous_standby_names``) will become empty after removal:
+            first remove nodes from ``voters`` and then from ``sync``.
+        else:
+            first remove nodes from ``sync`` and then from ``voters``.
+            Make ``voters`` empty if ``numsync_confirmed`` == ``0``.
+
+    :ivar leader: name of the leader, according to the ``/sync`` key.
+    :ivar quorum: ``quorum`` value from the ``/sync`` key, the minimal number of nodes we need see
+                  when doing the leader race.
+    :ivar voters: ``sync_standby`` value from the ``/sync`` key, set of node names we will be
+                  running the leader race against.
+    :ivar numsync: the number of synchronous nodes from the ``synchronous_standby_names``.
+    :ivar sync: set of node names listed in the ``synchronous_standby_names``.
+    :ivar numsync_confirmed: the number of nodes that are confirmed to reach "safe" LSN after they were added to the
+                  ``synchronous_standby_names``.
+    :ivar active: set of node names that are replicating from the primary (according to ``pg_stat_replication``)
+                  and are eligible to be listed in ``synchronous_standby_names``.
+    :ivar sync_wanted: desired number of synchronous nodes (``synchronous_node_count`` from the global configuration).
+    :ivar leader_wanted: the desired leader (could be different from the :attr:`leader` right after a failover).
+    """
+
+    def __init__(self, leader: str, quorum: int, voters: Collection[str],
+                 numsync: int, sync: Collection[str], numsync_confirmed: int,
+                 active: Collection[str], sync_wanted: int, leader_wanted: str) -> None:
+        """Instantiate :class:``QuorumStateResolver`` based on input parameters.
+
+        :param leader: name of the leader, according to the ``/sync`` key.
+        :param quorum: ``quorum`` value from the ``/sync`` key, the minimal number of nodes we need see
+                        when doing the leader race.
+        :param voters: ``sync_standby`` value from the ``/sync`` key, set of node names we will be
+                       running the leader race against.
+        :param numsync: the number of synchronous nodes from the ``synchronous_standby_names``.
+        :param sync: Set of node names listed in the ``synchronous_standby_names``.
+        :param numsync_confirmed: the number of nodes that are confirmed to reach "safe" LSN after
+                                  they were added to the ``synchronous_standby_names``.
+        :param active: set of node names that are replicating from the primary (according to ``pg_stat_replication``)
+                       and are eligible to be listed in ``synchronous_standby_names``.
+        :param sync_wanted: desired number of synchronous nodes
+                            (``synchronous_node_count`` from the global configuration).
+        :param leader_wanted: the desired leader (could be different from the *leader* right after a failover).
+
+        """
+        self.leader = leader
+        self.quorum = quorum
+        self.voters = CaseInsensitiveSet(voters)
+        self.numsync = min(numsync, len(sync))  # numsync can't be bigger than number of listed synchronous nodes.
+        self.sync = CaseInsensitiveSet(sync)
+        self.numsync_confirmed = numsync_confirmed
+        self.active = CaseInsensitiveSet(active)
+        self.sync_wanted = sync_wanted
+        self.leader_wanted = leader_wanted
+
+    def check_invariants(self) -> None:
+        """Checks invariant of ``synchronous_standby_names`` and ``/sync`` key in DCS.
+
+        .. seealso::
+            Check :class:`QuorumStateResolver`'s docstring for more information.
+
+        :raises:
+            :exc:`QuorumError`: in case of broken state"""
+        voters = CaseInsensitiveSet(self.voters | CaseInsensitiveSet([self.leader]))
+        sync = CaseInsensitiveSet(self.sync | CaseInsensitiveSet([self.leader_wanted]))
+
+        # We need to verify that subset of nodes that can acknowledge a commit overlaps
+        # with any subset of nodes that can achieve quorum to promote a new leader.
+        # ``+ 1`` is required because the leader is included in the set.
+        if self.voters and not (len(voters | sync) <= self.quorum + self.numsync + 1):
+            len_nodes = len(voters | sync)
+            raise QuorumError("Quorum and sync not guaranteed to overlap: "
+                              f"nodes {len_nodes} >= quorum {self.quorum} + sync {self.sync} + 1")
+        # unstable cases, we are changing synchronous_standby_names and /sync key
+        # one after another, hence one set is allowed to be a subset of another
+        if not (voters.issubset(sync) or sync.issubset(voters)):
+            voters_only = voters - sync
+            sync_only = sync - voters
+            raise QuorumError(f"Mismatched sets: voter only={voters_only} sync only={sync_only}")
+
+    def quorum_update(self, quorum: int, voters: CaseInsensitiveSet, leader: Optional[str] = None,
+                      adjust_quorum: Optional[bool] = True) -> Iterator[Transition]:
+        """Updates :attr:`quorum`, :attr:`voters` and optionally :attr:`leader` fields.
+
+        :param quorum: the new value for :attr:`quorum`, could be adjusted depending
+                       on values of :attr:`numsync_confirmed` and *adjust_quorum*.
+        :param voters: the new value for :attr:`voters`, could be adjusted if :attr:`numsync_confirmed` == ``0``.
+        :param leader: the new value for :attr:`leader`, optional.
+        :param adjust_quorum: if set to ``True`` the quorum requirement will be increased by the
+                              difference between :attr:`numsync` and :attr:`numsync_confirmed`.
+
+        :yields: the new state of the ``/sync`` key as a :class:`Transition` object.
+
+        :raises:
+            :exc:`QuorumError` in case of invalid data or if the invariant after transition could not be satisfied.
+        """
+        if quorum < 0:
+            raise QuorumError(f'Quorum {quorum} < 0 of ({voters})')
+        if quorum > 0 and quorum >= len(voters):
+            raise QuorumError(f'Quorum {quorum} >= N of ({voters})')
+
+        old_leader = self.leader
+        if leader is not None:  # Change of leader was requested
+            self.leader = leader
+        elif self.numsync_confirmed == 0:
+            # If there are no nodes that known to caught up with the primary we want to reset quorum/voters in /sync key
+            quorum = 0
+            voters = CaseInsensitiveSet()
+        elif adjust_quorum:
+            # It could be that the number of nodes that are known to catch up with the primary is below desired numsync.
+            # We want to increase quorum to guarantee that the sync node will be found during the leader race.
+            quorum += max(self.numsync - self.numsync_confirmed, 0)
+
+        if (self.leader, quorum, voters) == (old_leader, self.quorum, self.voters):
+            if self.voters:
+                return
+            # If transition produces no change of leader/quorum/voters we want to give a hint to
+            # the caller to fetch the new state from the database and restart QuorumStateResolver.
+            yield Transition('restart', self.leader, self.quorum, self.voters)
+
+        self.quorum = quorum
+        self.voters = voters
+        self.check_invariants()
+        logger.debug('quorum %s %s %s', self.leader, self.quorum, self.voters)
+        yield Transition('quorum', self.leader, self.quorum, self.voters)
+
+    def sync_update(self, numsync: int, sync: CaseInsensitiveSet) -> Iterator[Transition]:
+        """Updates :attr:`numsync` and :attr:`sync` fields.
+
+        :param numsync: the new value for :attr:`numsync`.
+        :param sync: the new value for :attr:`sync`:
+
+        :yields: the new state of ``synchronous_standby_names`` as a :class:`Transition` object.
+
+        :raises:
+            :exc:`QuorumError` in case of invalid data or if invariant after transition could not be satisfied
+        """
+        if numsync < 0:
+            raise QuorumError(f'Sync {numsync} < 0 of ({sync})')
+        if numsync > len(sync):
+            raise QuorumError(f'Sync {numsync} > N of ({sync})')
+
+        self.numsync = numsync
+        self.sync = sync
+        self.check_invariants()
+        logger.debug('sync %s %s %s', self.leader, self.numsync, self.sync)
+        yield Transition('sync', self.leader, self.numsync, self.sync)
+
+    def __iter__(self) -> Iterator[Transition]:
+        """Iterate over the transitions produced by :meth:`_generate_transitions`.
+
+        .. note::
+            Merge two transitions of the same type to a single one.
+
+            This is always safe because skipping the first transition is equivalent
+            to no one observing the intermediate state.
+
+        :yields: transitions as :class:`Transition` objects.
+        """
+        transitions = list(self._generate_transitions())
+        for cur_transition, next_transition in zip(transitions, transitions[1:] + [None]):
+            if isinstance(next_transition, Transition) \
+                    and cur_transition.transition_type == next_transition.transition_type:
+                continue
+            yield cur_transition
+            if cur_transition.transition_type == 'restart':
+                break
+
+    def __handle_non_steady_cases(self) -> Iterator[Transition]:
+        """Handle cases when set of transitions produced on previous run was interrupted.
+
+        :yields: transitions as :class:`Transition` objects.
+        """
+        if self.sync < self.voters:
+            logger.debug("Case 1: synchronous_standby_names %s is a subset of DCS state %s", self.sync, self.voters)
+            # Case 1: voters is superset of sync nodes. In the middle of changing voters (quorum).
+            # Evict  dead nodes from voters that are not being synced.
+            remove_from_voters = self.voters - (self.sync | self.active)
+            if remove_from_voters:
+                yield from self.quorum_update(
+                    quorum=len(self.voters) - len(remove_from_voters) - self.numsync,
+                    voters=CaseInsensitiveSet(self.voters - remove_from_voters),
+                    adjust_quorum=not (self.sync - self.active))
+            # Start syncing to nodes that are in voters and alive
+            add_to_sync = (self.voters & self.active) - self.sync
+            if add_to_sync:
+                yield from self.sync_update(self.numsync, CaseInsensitiveSet(self.sync | add_to_sync))
+        elif self.sync > self.voters:
+            logger.debug("Case 2: synchronous_standby_names %s is a superset of DCS state %s", self.sync, self.voters)
+            # Case 2: sync is superset of voters nodes. In the middle of changing replication factor (sync).
+            # Add to voters nodes that are already synced and active
+            add_to_voters = (self.sync - self.voters) & self.active
+            if add_to_voters:
+                voters = CaseInsensitiveSet(self.voters | add_to_voters)
+                yield from self.quorum_update(len(voters) - self.numsync, voters)
+            # Remove from sync nodes that are dead
+            remove_from_sync = self.sync - self.voters
+            if remove_from_sync:
+                yield from self.sync_update(
+                    numsync=min(self.numsync, len(self.sync) - len(remove_from_sync)),
+                    sync=CaseInsensitiveSet(self.sync - remove_from_sync))
+
+        # After handling these two cases voters and sync must match.
+        assert self.voters == self.sync
+
+        safety_margin = self.quorum + min(self.numsync, self.numsync_confirmed) - len(self.voters | self.sync)
+        if safety_margin > 0:  # In the middle of changing replication factor.
+            if self.numsync > self.sync_wanted:
+                numsync = max(self.sync_wanted, len(self.voters) - self.quorum)
+                logger.debug('Case 3: replication factor %d is bigger than needed %d', self.numsync, numsync)
+                yield from self.sync_update(numsync, self.sync)
+            else:
+                quorum = len(self.sync) - self.numsync
+                logger.debug('Case 4: quorum %d is bigger than needed %d', self.quorum, quorum)
+                yield from self.quorum_update(quorum, self.voters)
+        else:
+            safety_margin = self.quorum + self.numsync - len(self.voters | self.sync)
+            if self.numsync == self.sync_wanted and safety_margin > 0 and self.numsync > self.numsync_confirmed:
+                yield from self.quorum_update(len(self.sync) - self.numsync, self.voters)
+
+    def __remove_gone_nodes(self) -> Iterator[Transition]:
+        """Remove inactive nodes from ``synchronous_standby_names`` and from ``/sync`` key.
+
+        :yields: transitions as :class:`Transition` objects.
+        """
+        to_remove = self.sync - self.active
+        if to_remove and self.sync == to_remove:
+            logger.debug("Removing nodes: %s", to_remove)
+            yield from self.quorum_update(0, CaseInsensitiveSet(), adjust_quorum=False)
+            yield from self.sync_update(0, CaseInsensitiveSet())
+        elif to_remove:
+            logger.debug("Removing nodes: %s", to_remove)
+            can_reduce_quorum_by = self.quorum
+            # If we can reduce quorum size try to do so first
+            if can_reduce_quorum_by:
+                # Pick nodes to remove by sorted order to provide deterministic behavior for tests
+                remove = CaseInsensitiveSet(sorted(to_remove, reverse=True)[:can_reduce_quorum_by])
+                sync = CaseInsensitiveSet(self.sync - remove)
+                # when removing nodes from sync we can safely increase numsync if requested
+                numsync = min(self.sync_wanted, len(sync)) if self.sync_wanted > self.numsync else self.numsync
+                yield from self.sync_update(numsync, sync)
+                voters = CaseInsensitiveSet(self.voters - remove)
+                to_remove &= self.sync
+                yield from self.quorum_update(len(voters) - self.numsync, voters,
+                                              adjust_quorum=not to_remove)
+            if to_remove:
+                assert self.quorum == 0
+                numsync = self.numsync - len(to_remove)
+                sync = CaseInsensitiveSet(self.sync - to_remove)
+                voters = CaseInsensitiveSet(self.voters - to_remove)
+                sync_decrease = numsync - min(self.sync_wanted, len(sync))
+                quorum = min(sync_decrease, len(voters) - 1) if sync_decrease else 0
+                yield from self.quorum_update(quorum, voters, adjust_quorum=False)
+                yield from self.sync_update(numsync, sync)
+
+    def __add_new_nodes(self) -> Iterator[Transition]:
+        """Add new active nodes to ``synchronous_standby_names`` and to ``/sync`` key.
+
+        :yields: transitions as :class:`Transition` objects.
+        """
+        to_add = self.active - self.sync
+        if to_add:
+            # First get to requested replication factor
+            logger.debug("Adding nodes: %s", to_add)
+            sync_wanted = min(self.sync_wanted, len(self.sync | to_add))
+            increase_numsync_by = sync_wanted - self.numsync
+            if increase_numsync_by > 0:
+                if self.sync:
+                    add = CaseInsensitiveSet(sorted(to_add)[:increase_numsync_by])
+                    increase_numsync_by = len(add)
+                else:  # there is only the leader
+                    add = to_add  # and it is safe to add all nodes at once if sync is empty
+                yield from self.sync_update(self.numsync + increase_numsync_by, CaseInsensitiveSet(self.sync | add))
+                voters = CaseInsensitiveSet(self.voters | add)
+                yield from self.quorum_update(len(voters) - sync_wanted, voters)
+                to_add -= self.sync
+            if to_add:
+                voters = CaseInsensitiveSet(self.voters | to_add)
+                yield from self.quorum_update(len(voters) - sync_wanted, voters,
+                                              adjust_quorum=sync_wanted > self.numsync_confirmed)
+                yield from self.sync_update(sync_wanted, CaseInsensitiveSet(self.sync | to_add))
+
+    def __handle_replication_factor_change(self) -> Iterator[Transition]:
+        """Handle change of the replication factor (:attr:`sync_wanted`, aka ``synchronous_node_count``).
+
+        :yields: transitions as :class:`Transition` objects.
+        """
+        # Apply requested replication factor change
+        sync_increase = min(self.sync_wanted, len(self.sync)) - self.numsync
+        if sync_increase > 0:
+            # Increase replication factor
+            logger.debug("Increasing replication factor to %s", self.numsync + sync_increase)
+            yield from self.sync_update(self.numsync + sync_increase, self.sync)
+            yield from self.quorum_update(len(self.voters) - self.numsync, self.voters)
+        elif sync_increase < 0:
+            # Reduce replication factor
+            logger.debug("Reducing replication factor to %s", self.numsync + sync_increase)
+            if self.quorum - sync_increase < len(self.voters):
+                yield from self.quorum_update(len(self.voters) - self.numsync - sync_increase, self.voters,
+                                              adjust_quorum=self.sync_wanted > self.numsync_confirmed)
+            yield from self.sync_update(self.numsync + sync_increase, self.sync)
+
+    def _generate_transitions(self) -> Iterator[Transition]:
+        """Produce a set of changes to safely transition from the current state to the desired.
+
+        :yields: transitions as :class:`Transition` objects.
+        """
+        logger.debug("Quorum state: leader %s quorum %s, voters %s, numsync %s, sync %s, "
+                     "numsync_confirmed %s, active %s, sync_wanted %s leader_wanted %s",
+                     self.leader, self.quorum, self.voters, self.numsync, self.sync,
+                     self.numsync_confirmed, self.active, self.sync_wanted, self.leader_wanted)
+        try:
+            if self.leader_wanted != self.leader:  # failover
+                voters = (self.voters - CaseInsensitiveSet([self.leader_wanted])) | CaseInsensitiveSet([self.leader])
+                if not self.sync:
+                    # If sync is empty we need to update synchronous_standby_names first
+                    numsync = len(voters) - self.quorum
+                    yield from self.sync_update(numsync, CaseInsensitiveSet(voters))
+                # If leader changed we need to add the old leader to quorum (voters)
+                yield from self.quorum_update(self.quorum, CaseInsensitiveSet(voters), self.leader_wanted)
+                # right after promote there could be no replication connections yet
+                if not self.sync & self.active:
+                    return  # give another loop_wait seconds for replicas to reconnect before removing them from quorum
+            else:
+                self.check_invariants()
+        except QuorumError as e:
+            logger.warning('%s', e)
+            yield from self.quorum_update(len(self.sync) - self.numsync, self.sync)
+
+        assert self.leader == self.leader_wanted
+
+        # numsync_confirmed could be 0 after restart/failover, we will calculate it from quorum
+        if self.numsync_confirmed == 0 and self.sync & self.active:
+            self.numsync_confirmed = min(len(self.sync & self.active), len(self.voters) - self.quorum)
+            logger.debug('numsync_confirmed=0, adjusting it to %d', self.numsync_confirmed)
+
+        yield from self.__handle_non_steady_cases()
+
+        # We are in a steady state point. Find if desired state is different and act accordingly.
+
+        yield from self.__remove_gone_nodes()
+
+        yield from self.__add_new_nodes()
+
+        yield from self.__handle_replication_factor_change()

patroni/request.py

@@ -11,6 +11,19 @@ from .dcs import Member
 from .utils import USER_AGENT
 
 
+class HTTPSConnectionPool(urllib3.HTTPSConnectionPool):
+
+    def _validate_conn(self, *args: Any, **kwargs: Any) -> None:
+        """Override parent method to silence warnings about requests without certificate verification enabled."""
+
+
+class PatroniPoolManager(urllib3.PoolManager):
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super(PatroniPoolManager, self).__init__(*args, **kwargs)
+        self.pool_classes_by_scheme = {'http': urllib3.HTTPConnectionPool, 'https': HTTPSConnectionPool}
+
+
 class PatroniRequest(object):
     """Wrapper for performing requests to Patroni's REST API.
 
@@ -21,29 +34,38 @@ class PatroniRequest(object):
         """Create a new :class:`PatroniRequest` instance with given *config*.
 
         :param config: Patroni YAML configuration.
-        :param insecure: how to deal with SSL certs verification
+        :param insecure: how to deal with SSL certs verification:
+
             * If ``True`` it will perform REST API requests without verifying SSL certs; or
             * If ``False`` it will perform REST API requests and verify SSL certs; or
-            * If ``None`` it will behave according to the value of ``ctl -> insecure`` configuration; or
+            * If ``None`` it will behave according to the value of ``ctl.insecure`` configuration; or
             * If none of the above applies, then it falls back to ``False``.
         """
         self._insecure = insecure
-        self._pool = urllib3.PoolManager(num_pools=10, maxsize=10)
+        self._pool = PatroniPoolManager(num_pools=10, maxsize=10)
         self.reload_config(config)
 
     @staticmethod
-    def _get_cfg_value(config: Union[Config, Dict[str, Any]], name: str) -> Union[Any, None]:
+    def _get_ctl_value(config: Union[Config, Dict[str, Any]], name: str, default: Any = None) -> Optional[Any]:
-        """Get value of *name* setting in *config*.
+        """Get value of *name* setting from the ``ctl`` section of the *config*.
-
-        .. note::
-            *name* key will be searched only under ``ctl`` and ``restapi`` sections, in that order.
 
         :param config: Patroni YAML configuration.
         :param name: name of the setting value to be retrieved.
 
-        :returns: value of ``ctl -> *name*`` or ``restapi -> *name*``, if either is present, ``None`` otherwise.
+        :returns: value of ``ctl.*name*`` if present, ``None`` otherwise.
         """
-        return config.get('ctl', {}).get(name) or config.get('restapi', {}).get(name)
+        return config.get('ctl', {}).get(name, default)
+
+    @staticmethod
+    def _get_restapi_value(config: Union[Config, Dict[str, Any]], name: str) -> Optional[Any]:
+        """Get value of *name* setting from the ``restapi`` section of the *config*.
+
+        :param config: Patroni YAML configuration.
+        :param name: name of the setting value to be retrieved.
+
+        :returns: value of ``restapi -> *name*`` if present, ``None`` otherwise.
+        """
+        return config.get('restapi', {}).get(name)
 
     def _apply_pool_param(self, param: str, value: Any) -> None:
         """Configure *param* as *value* in the request manager.
@@ -62,15 +84,15 @@ class PatroniRequest(object):
 
         :param config: Patroni YAML configuration.
         :param name: prefix of the Patroni SSL related setting name. Currently, supports these:
+
             * ``cert``: gets translated to ``certfile``
             * ``key``: gets translated to ``keyfile``
 
-            Will attempt to fetch the requested key first from ``ctl`` section, and fall back to ``restapi`` section
+            Will attempt to fetch the requested key first from ``ctl`` section.
-            if the former is missing.
 
-        :returns: value of ``ctl -> *name*file`` or ``restapi -> *name*file`` if either is present, ``None`` otherwise.
+        :returns: value of ``ctl.*name*file`` if present, ``None`` otherwise.
         """
-        value = self._get_cfg_value(config, name + 'file')
+        value = self._get_ctl_value(config, name + 'file')
         self._apply_pool_param(name + '_file', value)
         return value
 
@@ -79,37 +101,39 @@ class PatroniRequest(object):
 
         Configure these HTTP headers for requests:
 
-        * ``authorization``: based on Patroni' REST API authentication config;
+            * ``authorization``: based on Patroni' CTL or REST API authentication config;
-        * ``user-agent``: based on `patroni.utils.USER_AGENT`.
+            * ``user-agent``: based on ``patroni.utils.USER_AGENT``.
 
         Also configure SSL related settings for requests:
 
-        * ``ca_certs`` is configured if ``ctl -> cacert`` or ``restapi -> cafile`` is available;
+            * ``ca_certs`` is configured if ``ctl.cacert`` or ``restapi.cafile`` is available;
-        * ``cert``, ``key`` and ``key_password`` are configured if ``ctl -> certile`` or ``restapi -> certfile`` is
+            * ``cert``, ``key`` and ``key_password`` are configured if ``ctl.certfile`` is available.
-            available.
 
         :param config: Patroni YAML configuration.
         """
-        # ``restapi -> auth`` is equivalent to ``restapi -> authentication -> username`` + ``:`` +
+        # ``ctl -> auth`` is equivalent to ``ctl -> authentication -> username`` + ``:`` +
-        # ``restapi -> authentication -> password``
+        # ``ctl -> authentication -> password``. And the same for ``restapi -> auth``
-        self._pool.headers = urllib3.make_headers(basic_auth=self._get_cfg_value(config, 'auth'), user_agent=USER_AGENT)
+        basic_auth = self._get_ctl_value(config, 'auth') or self._get_restapi_value(config, 'auth')
+        self._pool.headers = urllib3.make_headers(basic_auth=basic_auth, user_agent=USER_AGENT)
+        self._pool.connection_pool_kw['cert_reqs'] = 'CERT_REQUIRED'
+
+        insecure = self._insecure if isinstance(self._insecure, bool)\
+            else self._get_ctl_value(config, 'insecure', False)
 
-        insecure = self._insecure if isinstance(self._insecure, bool) else config.get('ctl', {}).get('insecure', False)
         if self._apply_ssl_file_param(config, 'cert'):
-            #  With client certificate the cert_reqs must be set to CERT_REQUIRED even if insecure option is used
+            if insecure:  # The assert_hostname = False helps to silence warnings
-            self._pool.connection_pool_kw['cert_reqs'] = 'CERT_REQUIRED'
+                self._pool.connection_pool_kw['assert_hostname'] = False
-            #  The assert_hostname = False helps to silence warnings
-            self._pool.connection_pool_kw['assert_hostname'] = False if insecure else None
 
             self._apply_ssl_file_param(config, 'key')
-
+            password = self._get_ctl_value(config, 'keyfile_password')
-            password = self._get_cfg_value(config, 'keyfile_password')
             self._apply_pool_param('key_password', password)
         else:
-            self._pool.connection_pool_kw['cert_reqs'] = 'CERT_NONE' if insecure else 'CERT_REQUIRED'
+            if insecure:  # Disable server certificate validation if requested
+                self._pool.connection_pool_kw['cert_reqs'] = 'CERT_NONE'
+            self._pool.connection_pool_kw.pop('assert_hostname', None)
             self._pool.connection_pool_kw.pop('key_file', None)
 
-        cacert = config.get('ctl', {}).get('cacert') or config.get('restapi', {}).get('cafile')
+        cacert = self._get_ctl_value(config, 'cacert') or self._get_restapi_value(config, 'cafile')
         self._apply_pool_param('ca_certs', cacert)
 
     def request(self, method: str, url: str, body: Optional[Any] = None,

patroni/scripts/barman/__init__.py

@@ -0,0 +1 @@
+"""Create :mod:`patroni.scripts.barman`."""