Reword confirmation message

- Implement the dedicated class that represents a manual failover
  request
- Move manual failover/switchover prechecks to the class method and use
  for both ctl and api
- Use a single parse_schedule function in both ctl and api
- Implement has_members_eligible_to_promote Ha method
- Fix get_members + role='any' exception msg

.github/workflows/tests.yaml

@@ -173,4 +173,28 @@ jobs:
 
     - uses: jakebailey/pyright-action@v1
       with:
-        version: 1.1.317
+        version: 1.1.320
+
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.11
+        cache: pip
+
+    - name: Install dependencies
+      run: pip install tox
+
+    - name: Install package dependencies
+      run: |
+        sudo apt update \
+        && sudo apt install -y \
+            latexmk texlive-latex-extra tex-gyre \
+            --no-install-recommends
+
+    - name: Generate documentation
+      run: tox -m docs

.gitignore

@@ -33,7 +33,7 @@ nosetests.xml
 coverage.xml
 htmlcov
 junit.xml
-features/output
+features/output*
 dummy
 
 # Translations
@@ -48,9 +48,12 @@ pgpass
 scm-source.json
 
 # Sphinx-generated documentation
+docs/_build/
 docs/build/
 docs/source/_static/
 docs/source/_templates/
+docs/modules/
+docs/pdf/
 
 # Pycharm IDE
 .idea/
@@ -63,3 +66,6 @@ venv*/
 
 # Default test data directory
 data/
+
+# macOS
+**/.DS_Store

.readthedocs.yaml

@@ -19,3 +19,8 @@ formats:
   - epub
   - pdf
   - htmlzip
+
+python:
+  install:
+    - requirements: requirements.docs.txt
+    - requirements: requirements.txt

Dockerfile

@@ -25,8 +25,7 @@ RUN set -ex \
             | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \
             | xargs apt-get install -y vim curl less jq locales haproxy sudo \
                             python3-etcd python3-kazoo python3-pip busybox \
-                            net-tools iputils-ping --fix-missing \
+                            net-tools iputils-ping dumb-init --fix-missing \
-    && pip3 install dumb-init \
 \
     # Cleanup all locales but en_US.UTF-8
     && find /usr/share/i18n/charmaps/ -type f ! -name UTF-8.gz -delete \
@@ -71,7 +70,7 @@ RUN set -ex \
     # Clean up all useless packages and some files
     && apt-get purge -y --allow-remove-essential python3-pip gzip bzip2 util-linux e2fsprogs \
                 libmagic1 bsdmainutils login ncurses-bin libmagic-mgc e2fslibs bsdutils \
-                exim4-config gnupg-agent dirmngr libpython2.7-stdlib libpython2.7-minimal \
+                exim4-config gnupg-agent dirmngr \
                 git make \
     && apt-get autoremove -y \
     && apt-get clean -y \

Dockerfile.citus

@@ -25,7 +25,7 @@ RUN set -ex \
             | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \
             | xargs apt-get install -y vim curl less jq locales haproxy sudo \
                             python3-etcd python3-kazoo python3-pip busybox \
-                            net-tools iputils-ping lsb-release --fix-missing \
+                            net-tools iputils-ping lsb-release dumb-init --fix-missing \
     && if [ $(dpkg --print-architecture) = 'arm64' ]; then \
         apt-get install -y postgresql-server-dev-$PG_MAJOR \
                            git gcc make autoconf \
@@ -42,7 +42,6 @@ RUN set -ex \
         && apt-get update -y \
         && apt-get -y install postgresql-$PG_MAJOR-citus-11.3; \
     fi \
-    && pip3 install dumb-init \
 \
     # Cleanup all locales but en_US.UTF-8
     && find /usr/share/i18n/charmaps/ -type f ! -name UTF-8.gz -delete \
@@ -88,7 +87,7 @@ RUN set -ex \
     # Clean up all useless packages and some files
     && apt-get purge -y --allow-remove-essential python3-pip gzip bzip2 util-linux e2fsprogs \
                 libmagic1 bsdmainutils login ncurses-bin libmagic-mgc e2fslibs bsdutils \
-                exim4-config gnupg-agent dirmngr libpython2.7-stdlib libpython2.7-minimal \
+                exim4-config gnupg-agent dirmngr \
                 postgresql-server-dev-$PG_MAJOR git gcc make autoconf \
                 libc6-dev flex libicu-dev libkrb5-dev liblz4-dev \
                 libpam0g-dev libreadline-dev libselinux1-dev libssl-dev libxslt1-dev libzstd-dev uuid-dev \

README.rst

@@ -74,9 +74,8 @@ There are a few options available:
 
 ::
 
-    sudo apt-get install python-psycopg2   # install python2 psycopg2 module on Debian/Ubuntu
+    sudo apt-get install python3-psycopg2  # install psycopg2 module on Debian/Ubuntu
-    sudo apt-get install python3-psycopg2  # install python3 psycopg2 module on Debian/Ubuntu
+    sudo yum install python3-psycopg2      # install psycopg2 on RedHat/Fedora/CentOS
-    sudo yum install python-psycopg2       # install python2 psycopg2 on RedHat/Fedora/CentOS
 
 2. Install psycopg2 from the binary package
 
@@ -94,7 +93,7 @@ There are a few options available:
 
 ::
 
-    pip install psycopg[binary]
+    pip install psycopg[binary]>=3.0.0
 
 **General installation for pip**
 

docs/CONTRIBUTING.rst

@@ -1,182 +1,12 @@
 .. _contributing:
 
-Contributing guidelines
+Contributing
-=======================
+============
 
-Wanna contribute to Patroni? Yay - here is how!
+Resources and information for developers can be found in the pages below.
 
-Chatting
+.. toctree::
---------
+   :maxdepth: 2
 
-Just want to chat with other Patroni users?  Looking for interactive troubleshooting help? Join us on channel `#patroni <https://postgresteam.slack.com/archives/C9XPYG92A>`__ in the `PostgreSQL Slack <https://pgtreats.info/slack-invite>`__.
+   contributing_guidelines
-
+   Patroni API docs<modules/modules>
-Running tests
--------------
-
-Requirements for running behave tests:
-
-1. PostgreSQL packages need to be installed.
-2. PostgreSQL binaries must be available in your `PATH`. You may need to add them to the path with something like `PATH=/usr/lib/postgresql/11/bin:$PATH python -m behave`.
-3. If you'd like to test with external DCSs (e.g., Etcd, Consul, and Zookeeper) you'll need the packages installed and respective services running and accepting unencrypted/unprotected connections on localhost and default port. In the case of Etcd or Consul, the behave test suite could start them up if binaries are available in the `PATH`.
-
-Install dependencies:
-
-.. code-block:: bash
-
-    # You may want to use Virtualenv or specify pip3.
-    pip install -r requirements.txt
-    pip install -r requirements.dev.txt
-
-After you have all dependencies installed, you can run the various test suites:
-
-.. code-block:: bash
-
-    # You may want to use Virtualenv or specify python3.
-
-    # Run flake8 to check syntax and formatting:
-    python setup.py flake8
-
-    # Run the pytest suite in tests/:
-    python setup.py test
-
-    # Run the behave (https://behave.readthedocs.io/en/latest/) test suite in features/;
-    # modify DCS as desired (raft has no dependencies so is the easiest to start with):
-    DCS=raft python -m behave
-
-Testing with tox
-----------------
-
-To run tox tests you only need to install one dependency (other than Python)
-
-.. code-block:: bash
-
-    pip install tox>=4
-
-If you wish to run `behave` tests then you also need docker installed.
-
-Tox configuration in `tox.ini` has "environments" to run the following tasks:
-
-* lint: Python code lint with `flake8`
-* test: unit tests for all available python interpreters with `pytest`,
-  generates XML reports or HTML reports if a TTY is detected
-* dep: detect package dependency conflicts using `pipdeptree`
-* type: static type checking with `pyright`
-* black: code formatting with `black`
-* docker-build: build docker image used for the `behave` env
-* docker-cmd: run arbitrary command with the above image
-* docker-behave-etcd: run tox for behave tests with above image
-* py*behave: run behave with available python interpreters (without docker, although
-  this is what is called inside docker containers)
-* docs: build docs with `sphinx`
-
-Running tox
-^^^^^^^^^^^
-
-To run the default env list; dep, lint, test, and docs, just run:
-
-.. code-block:: bash
-
-    tox
-
-The `test` envs can be run with the label `test`:
-
-.. code-block:: bash
-
-   tox -m test
-
-The `behave` docker tests can be run with the label `behave`:
-
-.. code-block:: bash
-
-   tox -m behave
-
-Similarly, docs has the label `docs`.
-
-All other envs can be run with their respective env names:
-
-.. code-block:: bash
-
-   tox -e lint
-   tox -e py39-test-lin
-
-It is also possible to select partial env lists using `factors`. For example, if you want to run
-all envs for python 3.10:
-
-.. code-block:: bash
-
-    tox -f py310
-
-This is equivalent to running all the envs listed below:
-
-.. code-block:: bash
-
-    $ tox -l -f py310
-    py310-test-lin
-    py310-test-mac
-    py310-test-win
-    py310-type-lin
-    py310-type-mac
-    py310-type-win
-    py310-behave-etcd-lin
-    py310-behave-etcd-win
-    py310-behave-etcd-mac
-
-
-You can list all configured combinations of environments with tox (>=v4) like so
-
-.. code-block:: bash
-
-    tox l
-
-The envs `test` and `docs` will attempt to open the HTML output files
-when the job completes, if tox is run with an active terminal. This
-is intended to be for benefit of the developer running this env locally.
-It will attempt to run `open` on a mac and `xdg-open` on Linux.
-To use a different command set the env var `OPEN_CMD` to the name or path of
-the command. If this step fails it will not fail the run overall.
-If you want to disable this facility set the env var `OPEN_CMD` to the `:` no-op command.
-
-.. code-block:: bash
-
-   OPEN_CMD=: tox -m docs
-
-Behave tests
-^^^^^^^^^^^^
-
-Behave tests with `-m behave` will build docker images based on PG_MAJOR version 11 through 15 and then run all
-behave tests. This can take quite a long time to run so you might want to limit the scope to a select version of
-Postgres or to a specific feature set or steps.
-
-To specify the version of postgres include the full name of the dependent image build env that you want and then the
-behave env name. For instance if you want Postgres 15 use:
-
-.. code-block:: bash
-
-    tox -e pg14-docker-build,pg14-docker-behave-etcd-lin
-
-If on the other hand you want to test a specific feature you can pass positional arguments to behave. This will run
-the watchdog behave feature test scenario with all versions of Postgres.
-
-.. code-block:: bash
-
-    tox -m behave -- features/watchdog.feature
-
-Of course you can combine the two.
-
-Reporting issues
-----------------
-
-If you have a question about patroni or have a problem using it, please read the :ref:`README <readme>` before filing an issue.
-Also double check with the current issues on our `Issues Tracker <https://github.com/zalando/patroni/issues>`__.
-
-Contributing a pull request
----------------------------
-
-1) Submit a comment to the relevant issue or create a new issue describing your proposed change.
-2) Do a fork, develop and test your code changes.
-3) Include documentation
-4) Submit a pull request.
-
-You'll get feedback about your pull request as soon as possible.
-
-Happy Patroni hacking ;-)

docs/ENVIRONMENT.rst

@@ -112,7 +112,11 @@ Kubernetes
 -  **PATRONI\_KUBERNETES\_NAMESPACE**: (optional) Kubernetes namespace where the Patroni pod is running. Default value is `default`.
 -  **PATRONI\_KUBERNETES\_LABELS**: Labels in format ``{label1: value1, label2: value2}``. These labels will be used to find existing objects (Pods and either Endpoints or ConfigMaps) associated with the current cluster. Also Patroni will set them on every object (Endpoint or ConfigMap) it creates.
 -  **PATRONI\_KUBERNETES\_SCOPE\_LABEL**: (optional) name of the label containing cluster name. Default value is `cluster-name`.
--  **PATRONI\_KUBERNETES\_ROLE\_LABEL**: (optional) name of the label containing Postgres role (`master` or `replica`). Patroni will set this label on the pod it is running in. Default value is `role`.
+-  **PATRONI\_KUBERNETES\_ROLE\_LABEL**: (optional) name of the label containing role (master or replica or other custom value). Patroni will set this label on the pod it runs in. Default value is ``role``.
+-  **PATRONI\_KUBERNETES\_LEADER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is `master`. Default value is `master`.
+-  **PATRONI\_KUBERNETES\_FOLLOWER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is `replica`. Default value is `replica`.
+-  **PATRONI\_KUBERNETES\_STANDBY\_LEADER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is ``standby_leader``. Default value is ``master``.
+-  **PATRONI\_KUBERNETES\_TMP\_ROLE\_LABEL**: (optional) name of the temporary label containing role (master or replica). Value of this label will always use the default of corresponding role. Set only when necessary.
 -  **PATRONI\_KUBERNETES\_USE\_ENDPOINTS**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
 -  **PATRONI\_KUBERNETES\_POD\_IP**: (optional) IP address of the pod Patroni is running in. This value is required when `PATRONI_KUBERNETES_USE_ENDPOINTS` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
 -  **PATRONI\_KUBERNETES\_PORTS**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``PATRONI_KUBERNETES_PORTS='[{"name": "postgresql", "port": 5432}]'`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `PATRONI_KUBERNETES_USE_ENDPOINTS` is set.
@@ -196,10 +200,19 @@ REST API
 -  **PATRONI\_RESTAPI\_HTTPS\_EXTRA\_HEADERS**: (optional) HTTPS headers let the REST API server pass additional information with an HTTP response when TLS is enabled. This will also pass additional information set in ``http_extra_headers``.
 -  **PATRONI\_RESTAPI\_REQUEST\_QUEUE\_SIZE**: (optional): Sets request queue size for TCP socket used by Patroni REST API.  Once the queue is full, further requests get a "Connection denied" error. The default value is 5.
 
+.. warning::
+
+    - The ``PATRONI_RESTAPI_CONNECT_ADDRESS`` must be accessible from all nodes of a given Patroni cluster. Internally Patroni is using it during the leader race to find nodes with minimal replication lag.
+    - If you enabled client certificates validation (``PATRONI_RESTAPI_VERIFY_CLIENT`` is set to ``required``), you also **must** provide **valid client certificates** in the ``PATRONI_CTL_CERTFILE``, ``PATRONI_CTL_KEYFILE``, ``PATRONI_CTL_KEYFILE_PASSWORD``. If not provided, Patroni will not work correctly.
+
+
 CTL
 ---
--  **PATRONICTL\_CONFIG\_FILE**: location of the configuration file.
+-  **PATRONICTL\_CONFIG\_FILE**: (optional) location of the configuration file.
--  **PATRONI\_CTL\_INSECURE**: Allow connections to REST API without verifying SSL certs.
+-  **PATRONI\_CTL\_USERNAME**: (optional) Basic-auth username for accessing protected REST API endpoints. If not provided patronictl will use the value provided for REST API "username" parameter.
--  **PATRONI\_CTL\_CACERT**: Specifies the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs. If not provided patronictl will use the value provided for REST API "cafile" parameter.
+-  **PATRONI\_CTL\_PASSWORD**: (optional) Basic-auth password for accessing protected REST API endpoints. If not provided patronictl will use the value provided for REST API "password" parameter.
--  **PATRONI\_CTL\_CERTFILE**: Specifies the file with the client certificate in the PEM format. If not provided patronictl will use the value provided for REST API "certfile" parameter.
+-  **PATRONI\_CTL\_INSECURE**: (optional) Allow connections to REST API without verifying SSL certs.
--  **PATRONI\_CTL\_KEYFILE**: Specifies the file with the client secret key in the PEM format. If not provided patronictl will use the value provided for REST API "keyfile" parameter.
+-  **PATRONI\_CTL\_CACERT**: (optional) Specifies the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs. If not provided patronictl will use the value provided for REST API "cafile" parameter.
+-  **PATRONI\_CTL\_CERTFILE**: (optional) Specifies the file with the client certificate in the PEM format.
+-  **PATRONI\_CTL\_KEYFILE**: (optional) Specifies the file with the client secret key in the PEM format.
+-  **PATRONI\_CTL\_KEYFILE\_PASSWORD**: (optional) Specifies a password for decrypting the client keyfile.

docs/README.rst

@@ -25,83 +25,7 @@ We report new releases information :ref:`here <releases>`.
 Technical Requirements/Installation
 -----------------------------------
 
-**Pre-requirements for Mac OS**
+Go :ref:`here <installation>` for guidance on installing and upgrading Patroni on various platforms.
-
-To install requirements on a Mac, run the following:
-
-::
-
-    brew install postgresql etcd haproxy libyaml python
-
-.. _psycopg2_install_options:
-
-**Psycopg**
-
-Starting from `psycopg2-2.8 <http://initd.org/psycopg/articles/2019/04/04/psycopg-28-released/>`__ the binary version of psycopg2 will no longer be installed by default. Installing it from the source code requires C compiler and postgres+python dev packages.
-Since in the python world it is not possible to specify dependency as ``psycopg2 OR psycopg2-binary`` you will have to decide how to install it.
-
-There are a few options available:
-
-1. Use the package manager from your distro
-
-::
-
-    sudo apt-get install python-psycopg2   # install python2 psycopg2 module on Debian/Ubuntu
-    sudo apt-get install python3-psycopg2  # install python3 psycopg2 module on Debian/Ubuntu
-    sudo yum install python-psycopg2       # install python2 psycopg2 on RedHat/Fedora/CentOS
-
-2. Install psycopg2 from the binary package
-
-::
-
-    pip install psycopg2-binary
-
-3. Install psycopg2 from source
-
-::
-
-    pip install psycopg2>=2.5.4
-
-4. Use psycopg 3.0 instead of psycopg2
-
-::
-
-    pip install psycopg[binary]>=3.0.0
-
-**General installation for pip**
-
-Patroni can be installed with pip:
-
-::
-
-    pip install patroni[dependencies]
-
-where dependencies can be either empty, or consist of one or more of the following:
-
-etcd or etcd3
-    `python-etcd` module in order to use Etcd as Distributed Configuration Store (DCS)
-consul
-    `python-consul` module in order to use Consul as DCS
-zookeeper
-    `kazoo` module in order to use Zookeeper as DCS
-exhibitor
-    `kazoo` module in order to use Exhibitor as DCS (same dependencies as for Zookeeper)
-kubernetes
-    `kubernetes` module in order to use Kubernetes as DCS in Patroni
-raft
-    `pysyncobj` module in order to use python Raft implementation as DCS
-aws
-    `boto3` in order to use AWS callbacks
-
-For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is:
-
-::
-
-    pip install patroni[etcd,aws]
-
-Note that external tools to call in the replica creation or custom bootstrap scripts (i.e. WAL-E) should be installed
-independently of Patroni.
-
 
 .. _running_configuring:
 
@@ -165,10 +89,6 @@ Applications Should Not Use Superusers
 
 When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
 
-.. |Build Status| image:: https://travis-ci.org/zalando/patroni.svg?branch=master
-   :target: https://travis-ci.org/zalando/patroni
-.. |Coverage Status| image:: https://coveralls.io/repos/zalando/patroni/badge.svg?branch=master
-   :target: https://coveralls.io/r/zalando/patroni?branch=master
 
 Testing Your HA Solution
 --------------------------------------

docs/citus.rst

@@ -140,7 +140,7 @@ An example of ``patronictl switchover`` on the worker cluster::
     | work2-1 | 172.27.0.5 | Sync Standby | running |  1 |         0 |
     | work2-2 | 172.27.0.7 | Leader       | running |  1 |           |
     +---------+------------+--------------+---------+----+-----------+
-    Are you sure you want to switchover cluster demo, demoting current primary work2-2? [y/N]: y
+    Are you sure you want to perform a switchover in the cluster demo, demoting current primary work2-2? [y/N]: y
     2022-12-22 07:02:40.33003 Successfully switched over to "work2-1"
     + Citus cluster: demo (group: 2, 7179854924063375386) ------+
     | Member  | Host       | Role    | State   | TL | Lag in MB |

docs/conf.py

@@ -20,10 +20,15 @@
 import os
 
 import sys
+
 sys.path.insert(0, os.path.abspath('..'))
 
 from patroni.version import __version__
 
+project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+module_dir = os.path.abspath(os.path.join(project_root, 'patroni'))
+excludes = ['tests', 'setup.py', 'conf']
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -33,11 +38,28 @@ from patroni.version import __version__
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['sphinx.ext.intersphinx',
+extensions = [
+    'sphinx.ext.intersphinx',
     'sphinx.ext.todo',
     'sphinx.ext.mathjax',
     'sphinx.ext.ifconfig',
-    'sphinx.ext.viewcode']
+    # 'sphinx.ext.viewcode',
+    'sphinx_github_style',  # Generate "View on GitHub" for source code
+    'sphinxcontrib.apidoc',  # For generating module docs from code
+    'sphinx.ext.autodoc',  # For generating module docs from docstrings
+    'sphinx.ext.napoleon',  # For Google and Numpy formatted docstrings
+]
+apidoc_module_dir = module_dir
+apidoc_output_dir = 'modules'
+apidoc_excluded_paths = excludes
+apidoc_separate_modules = True
+
+# Include autodoc for all members, including private ones and the ones that are missing a docstring.
+autodoc_default_options = {
+    "members": True,
+    "undoc-members": True,
+    "private-members": True,
+}
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -107,6 +129,34 @@ if not on_rtd:  # only import and set the theme if we're building docs locally
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
+# Replace "source" links with "edit on GitHub" when using rtd theme
+html_context = {
+    'display_github': True,
+    'github_user': 'zalando',
+    'github_repo': 'patroni',
+    'github_version': 'master',
+    'conf_py_path': '/docs/',
+}
+
+# sphinx-github-style options, https://sphinx-github-style.readthedocs.io/en/latest/index.html
+
+# The name of the top-level package.
+top_level = "patroni"
+
+# The blob to link to on GitHub - any of "head", "last_tag", or "{blob}"
+# linkcode_blob = 'head'
+
+# The link to your GitHub repository formatted as https://github.com/user/repo
+# If not provided, will attempt to create the link from the html_context dict
+# linkcode_url = f"https://github.com/{html_context['github_user']}/" \
+#                f"{html_context['github_repo']}/{html_context['github_version']}"
+
+# The text to use for the linkcode link
+# linkcode_link_text: str = "View on GitHub"
+
+# A linkcode_resolve() function to use for resolving the link target
+# linkcode_resolve: types.FunctionType
+
 
 # -- Options for HTMLHelp output ------------------------------------------
 
@@ -165,7 +215,6 @@ texinfo_documents = [
 ]
 
 
-
 # -- Options for Epub output ----------------------------------------------
 
 # Bibliographic Dublin Core info.
@@ -187,10 +236,65 @@ epub_copyright = copyright
 epub_exclude_files = ['search.html']
 
 
-
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {'python': ('https://docs.python.org/', None)}
 
+
+# Remove these pages from index, references, toc trees, etc.
+# If the builder is not 'html' then add the API docs modules index to pages to be removed.
+exclude_from_builder = {
+    'latex': ['modules/modules'],
+    'epub': ['modules/modules'],
+}
+# Internal holding list, anything added here will always be excluded
+_docs_to_remove = []
+
+
+def builder_inited(app):
+    """Run during Sphinx `builder-inited` phase.
+
+    Set a config value to builder name and add module docs to `docs_to_remove`.
+    """
+    print(f'The builder is: {app.builder.name}')
+    app.add_config_value('builder', app.builder.name, 'env')
+
+    # Remove pages when builder matches any referenced in exclude_from_builder
+    if exclude_from_builder.get(app.builder.name):
+        _docs_to_remove.extend(exclude_from_builder[app.builder.name])
+
+
+def env_get_outdated(app, env, added, changed, removed):
+    """Run during Sphinx `env-get-outdated` phase.
+
+    Remove the items listed in `docs_to_remove` from known pages.
+    """
+    added.difference_update(_docs_to_remove)
+    changed.difference_update(_docs_to_remove)
+    removed.update(_docs_to_remove)
+    return []
+
+
+def doctree_read(app, doctree):
+    """Run during Sphinx `doctree-read` phase.
+
+    Remove the items listed in `docs_to_remove` from the table of contents.
+    """
+    from sphinx import addnodes
+    for toc_tree_node in doctree.traverse(addnodes.toctree):
+        for e in toc_tree_node['entries']:
+            ref = str(e[1])
+            if ref in _docs_to_remove:
+                toc_tree_node['entries'].remove(e)
+
+
+def autodoc_skip(app, what, name, obj, would_skip, options):
+    """Include autodoc of ``__init__`` methods, which are skipped by default."""
+    if name == "__init__":
+        return False
+    return would_skip
+
+
+
 # A possibility to have an own stylesheet, to add new rules or override existing ones
 # For the latter case, the CSS specificity of the rules should be higher than the default ones
 def setup(app):
@@ -198,3 +302,9 @@ def setup(app):
         app.add_css_file('custom.css')
     else:
         app.add_stylesheet('custom.css')
+
+    # Run extra steps to remove module docs when running with a non-html builder
+    app.connect('builder-inited', builder_inited)
+    app.connect('env-get-outdated', env_get_outdated)
+    app.connect('doctree-read', doctree_read)
+    app.connect("autodoc-skip-member", autodoc_skip)

docs/contributing_guidelines.rst

@@ -0,0 +1,182 @@
+.. _contributing_guidelines:
+
+Contributing guidelines
+=======================
+
+Wanna contribute to Patroni? Yay - here is how!
+
+Chatting
+--------
+
+Just want to chat with other Patroni users?  Looking for interactive troubleshooting help? Join us on channel `#patroni <https://postgresteam.slack.com/archives/C9XPYG92A>`__ in the `PostgreSQL Slack <https://pgtreats.info/slack-invite>`__.
+
+Running tests
+-------------
+
+Requirements for running behave tests:
+
+1. PostgreSQL packages need to be installed.
+2. PostgreSQL binaries must be available in your `PATH`. You may need to add them to the path with something like `PATH=/usr/lib/postgresql/11/bin:$PATH python -m behave`.
+3. If you'd like to test with external DCSs (e.g., Etcd, Consul, and Zookeeper) you'll need the packages installed and respective services running and accepting unencrypted/unprotected connections on localhost and default port. In the case of Etcd or Consul, the behave test suite could start them up if binaries are available in the `PATH`.
+
+Install dependencies:
+
+.. code-block:: bash
+
+    # You may want to use Virtualenv or specify pip3.
+    pip install -r requirements.txt
+    pip install -r requirements.dev.txt
+
+After you have all dependencies installed, you can run the various test suites:
+
+.. code-block:: bash
+
+    # You may want to use Virtualenv or specify python3.
+
+    # Run flake8 to check syntax and formatting:
+    python setup.py flake8
+
+    # Run the pytest suite in tests/:
+    python setup.py test
+
+    # Run the behave (https://behave.readthedocs.io/en/latest/) test suite in features/;
+    # modify DCS as desired (raft has no dependencies so is the easiest to start with):
+    DCS=raft python -m behave
+
+Testing with tox
+----------------
+
+To run tox tests you only need to install one dependency (other than Python)
+
+.. code-block:: bash
+
+    pip install tox>=4
+
+If you wish to run `behave` tests then you also need docker installed.
+
+Tox configuration in `tox.ini` has "environments" to run the following tasks:
+
+* lint: Python code lint with `flake8`
+* test: unit tests for all available python interpreters with `pytest`,
+  generates XML reports or HTML reports if a TTY is detected
+* dep: detect package dependency conflicts using `pipdeptree`
+* type: static type checking with `pyright`
+* black: code formatting with `black`
+* docker-build: build docker image used for the `behave` env
+* docker-cmd: run arbitrary command with the above image
+* docker-behave-etcd: run tox for behave tests with above image
+* py*behave: run behave with available python interpreters (without docker, although
+  this is what is called inside docker containers)
+* docs: build docs with `sphinx`
+
+Running tox
+^^^^^^^^^^^
+
+To run the default env list; dep, lint, test, and docs, just run:
+
+.. code-block:: bash
+
+    tox
+
+The `test` envs can be run with the label `test`:
+
+.. code-block:: bash
+
+   tox -m test
+
+The `behave` docker tests can be run with the label `behave`:
+
+.. code-block:: bash
+
+   tox -m behave
+
+Similarly, docs has the label `docs`.
+
+All other envs can be run with their respective env names:
+
+.. code-block:: bash
+
+   tox -e lint
+   tox -e py39-test-lin
+
+It is also possible to select partial env lists using `factors`. For example, if you want to run
+all envs for python 3.10:
+
+.. code-block:: bash
+
+    tox -f py310
+
+This is equivalent to running all the envs listed below:
+
+.. code-block:: bash
+
+    $ tox -l -f py310
+    py310-test-lin
+    py310-test-mac
+    py310-test-win
+    py310-type-lin
+    py310-type-mac
+    py310-type-win
+    py310-behave-etcd-lin
+    py310-behave-etcd-win
+    py310-behave-etcd-mac
+
+
+You can list all configured combinations of environments with tox (>=v4) like so
+
+.. code-block:: bash
+
+    tox l
+
+The envs `test` and `docs` will attempt to open the HTML output files
+when the job completes, if tox is run with an active terminal. This
+is intended to be for benefit of the developer running this env locally.
+It will attempt to run `open` on a mac and `xdg-open` on Linux.
+To use a different command set the env var `OPEN_CMD` to the name or path of
+the command. If this step fails it will not fail the run overall.
+If you want to disable this facility set the env var `OPEN_CMD` to the `:` no-op command.
+
+.. code-block:: bash
+
+   OPEN_CMD=: tox -m docs
+
+Behave tests
+^^^^^^^^^^^^
+
+Behave tests with `-m behave` will build docker images based on PG_MAJOR version 11 through 15 and then run all
+behave tests. This can take quite a long time to run so you might want to limit the scope to a select version of
+Postgres or to a specific feature set or steps.
+
+To specify the version of postgres include the full name of the dependent image build env that you want and then the
+behave env name. For instance if you want Postgres 15 use:
+
+.. code-block:: bash
+
+    tox -e pg14-docker-build,pg14-docker-behave-etcd-lin
+
+If on the other hand you want to test a specific feature you can pass positional arguments to behave. This will run
+the watchdog behave feature test scenario with all versions of Postgres.
+
+.. code-block:: bash
+
+    tox -m behave -- features/watchdog.feature
+
+Of course you can combine the two.
+
+Reporting issues
+----------------
+
+If you have a question about patroni or have a problem using it, please read the :ref:`README <readme>` before filing an issue.
+Also double check with the current issues on our `Issues Tracker <https://github.com/zalando/patroni/issues>`__.
+
+Contributing a pull request
+---------------------------
+
+1) Submit a comment to the relevant issue or create a new issue describing your proposed change.
+2) Do a fork, develop and test your code changes.
+3) Include documentation
+4) Submit a pull request.
+
+You'll get feedback about your pull request as soon as possible.
+
+Happy Patroni hacking ;-)

docs/dynamic_configuration.rst

@@ -46,9 +46,9 @@ In order to change the dynamic configuration you can use either ``patronictl edi
    -  **archive\_cleanup\_command**: cleanup command for standby leader
    -  **recovery\_min\_apply\_delay**: how long to wait before actually apply WAL records on a standby leader
 
--  **slots**: define permanent replication slots. These slots will be preserved during switchover/failover. The logical slots are copied from the primary to a standby with restart, and after that their position advanced every **loop_wait** seconds (if necessary). Copying logical slot files performed via ``libpq`` connection and using either rewind or superuser credentials (see **postgresql.authentication** section). There is always a chance that the logical slot position on the replica is a bit behind the former primary, therefore application should be prepared that some messages could be received the second time after the failover. The easiest way of doing so - tracking ``confirmed_flush_lsn``. Enabling permanent logical replication slots requires **postgresql.use_slots** to be set and will also automatically enable the ``hot_standby_feedback``. Since the failover of logical replication slots is unsafe on PostgreSQL 9.6 and older and PostgreSQL version 10 is missing some important functions, the feature only works with PostgreSQL 11+.
+-  **slots**: define permanent replication slots. These slots will be preserved during switchover/failover. Permanent slots that don't exist will be created by Patroni. The physical slots are maintained only in the current primary. The logical slots are copied from the primary to a standby with restart, and after that their position advanced every **loop_wait** seconds (if necessary). Copying logical slot files performed via ``libpq`` connection and using either rewind or superuser credentials (see **postgresql.authentication** section). There is always a chance that the logical slot position on the replica is a bit behind the former primary, therefore application should be prepared that some messages could be received the second time after the failover. The easiest way of doing so - tracking ``confirmed_flush_lsn``. Enabling permanent logical replication slots requires **postgresql.use_slots** to be set and will also automatically enable the ``hot_standby_feedback``. Since the failover of logical replication slots is unsafe on PostgreSQL 9.6 and older and PostgreSQL version 10 is missing some important functions, the feature only works with PostgreSQL 11+.
 
-   -  **my\_slot\_name**: the name of replication slot. If the permanent slot name matches with the name of the current primary it will not be created. Everything else is the responsibility of the operator to make sure that there are no clashes in names between replication slots automatically created by Patroni for members and permanent replication slots.
+   -  **my\_slot\_name**: the name of the permanent replication slot. If the permanent slot name matches with the name of the current leader it will not be created. Please note that Patroni does not make checks for permanent slot names added to this configuration matching those that Patroni creates automatically for members. If those names are added, Patroni will ensure that any slots that were created are not removed even if the member becomes unresponsive, situation which would normally result in the slot's removal by Patroni. Although this can be useful in some situations, such as when importing existing members to a new Patroni cluster (see :ref:`Convert a Standalone to a Patroni Cluster <existing_data>` for details), caution should be exercised by the operator that these clashes in names are not persisted in the DCS due to its effect on normal functioning of Patroni.
 
       -  **type**: slot type. Could be ``physical`` or ``logical``. If the slot is logical, you have to additionally define ``database`` and ``plugin``.
       -  **database**: the database name where logical slots should be created.

docs/existing_data.rst

@@ -10,18 +10,58 @@ To deploy a Patroni cluster without using a pre-existing PostgreSQL instance, se
 Procedure
 ---------
 
-A Patroni cluster can be started with a data directory from a single-node PostgreSQL database. This is achieved by following closely these steps:
+You can find below an overview of steps for converting an existing Postgres cluster to a Patroni managed cluster. In the steps we assume all nodes that are part of the existing cluster are currently up and running, and that you *do not* intend to change Postgres configuration while the migration is ongoing. The steps:
 
-1. Manually start PostgreSQL daemon
+#. Create the Postgres users as explained for :ref:`authentication <postgresql_settings>` section of the Patroni configuration. You can find sample SQL commands to create the users in the code block below, in which you need to replace the usernames and passwords as per your environment. If you already have the relevant users, then you can skip this step.
-2. Create Patroni superuser and replication users as defined in the :ref:`authentication <postgresql_settings>` section of the Patroni configuration. If this user is created in SQL, the following queries achieve this:
 
-.. code-block:: sql
+   .. code-block:: sql
 
-   CREATE USER $PATRONI_SUPERUSER_USERNAME WITH SUPERUSER ENCRYPTED PASSWORD '$PATRONI_SUPERUSER_PASSWORD';
+      -- Patroni superuser
-   CREATE USER $PATRONI_REPLICATION_USERNAME WITH REPLICATION ENCRYPTED PASSWORD '$PATRONI_REPLICATION_PASSWORD';
+      -- Replace PATRONI_SUPERUSER_USERNAME and PATRONI_SUPERUSER_PASSWORD accordingly
+      CREATE USER PATRONI_SUPERUSER_USERNAME WITH SUPERUSER ENCRYPTED PASSWORD 'PATRONI_SUPERUSER_PASSWORD';
 
-3. Start Patroni (e.g. ``patroni /etc/patroni/patroni.yml``). It automatically detects that PostgreSQL daemon is already running but its configuration might be out-of-date.
+      -- Patroni replication user
-4. Ask Patroni to restart the node with ``patronictl restart cluster-name node-name``. This step is only required if PostgreSQL configuration is out-of-date.
+      -- Replace PATRONI_REPLICATION_USERNAME and PATRONI_REPLICATION_PASSWORD accordingly
+      CREATE USER PATRONI_REPLICATION_USERNAME WITH REPLICATION ENCRYPTED PASSWORD 'PATRONI_REPLICATION_PASSWORD';
+
+      -- Patroni rewind user, if you intend to enable use_pg_rewind in your Patroni configuration
+      -- Replace PATRONI_REWIND_USERNAME and PATRONI_REWIND_PASSWORD accordingly
+      CREATE USER PATRONI_REWIND_USERNAME WITH ENCRYPTED PASSWORD 'PATRONI_REWIND_PASSWORD';
+      GRANT EXECUTE ON function pg_catalog.pg_ls_dir(text, boolean, boolean) TO PATRONI_REWIND_USERNAME;
+      GRANT EXECUTE ON function pg_catalog.pg_stat_file(text, boolean) TO PATRONI_REWIND_USERNAME;
+      GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text) TO PATRONI_REWIND_USERNAME;
+      GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text, bigint, bigint, boolean) TO PATRONI_REWIND_USERNAME;
+
+#. Perform the following steps on all Postgres nodes. Perform all steps on one node before proceeding with the next node. Start with the primary node, then proceed with each standby node:
+
+   #. If you are running Postgres through systemd, then disable the Postgres systemd unit. This is performed as Patroni manages starting and stopping the Postgres daemon.
+
+   #. Create a YAML configuration file for Patroni.
+
+      * **Note (specific for the primary node):** If you have replication slots being used for replication between cluster members, then it is recommended that you enable ``use_slots`` and configure the existing replication slots as permanent via the ``slots`` configuration item. Be aware that Patroni automatically creates replication slots for replication between members, and drops replication slots that it does not recognize, when ``use_slots`` is enabled. The idea of using permanent slots here is to allow your existing slots to persist while the migration to Patroni is in progress. See :ref:`YAML Configuration Settings <yaml_configuration>` for details.
+
+   #. Start Patroni using the ``patroni`` systemd service unit. It automatically detects that Postgres is already running and starts monitoring the instance.
+
+#. Hand over Postgres "start up procedure" to Patroni. In order to do that you need to restart the cluster members through ``patronictl restart cluster-name member-name`` command. For minimal downtime you might want to split this step into:
+
+   #. Immediate restart of the standby nodes.
+   #. Scheduled restart of the primary node within a maintenance window.
+
+#. If you configured permanent slots in step ``1.2.``, then you should remove them from ``slots`` configuration through ``patronictl edit-config cluster-name member-name`` command once the ``restart_lsn`` of the slots created by Patroni is able to catch up with the ``restart_lsn`` of the original slots for the corresponding members. By removing the slots from ``slots`` configuration you will allow Patroni to drop the original slots from your cluster once they are not needed anymore. You can find below an example query to check the ``restart_lsn`` of a couple slots, so you can compare them:
+
+   .. code-block:: sql
+
+      -- Assume original_slot_for_member_x is the name of the slot in your original
+      -- cluster for replicating changes to member X, and slot_for_member_x is the
+      -- slot created by Patroni for that purpose. You need restart_lsn of
+      -- slot_for_member_x to be >= restart_lsn of original_slot_for_member_x
+      SELECT slot_name,
+             restart_lsn
+      FROM pg_replication_slots
+      WHERE slot_name IN (
+          'original_slot_for_member_x',
+          'slot_for_member_x'
+      )
 
 .. _major_upgrade:
 
@@ -30,14 +70,14 @@ Major Upgrade of PostgreSQL Version
 
 The only possible way to do a major upgrade currently is:
 
-1. Stop Patroni
+#. Stop Patroni
-2. Upgrade PostgreSQL binaries and perform `pg_upgrade <https://www.postgresql.org/docs/current/pgupgrade.html>`_ on the primary node
+#. Upgrade PostgreSQL binaries and perform `pg_upgrade <https://www.postgresql.org/docs/current/pgupgrade.html>`_ on the primary node
-3. Update patroni.yml
+#. Update patroni.yml
-4. Remove the initialize key from DCS or wipe complete cluster state from DCS. The second one could be achieved by running ``patronictl remove <cluster-name>``. It is necessary because pg_upgrade runs initdb which actually creates a new database with a new PostgreSQL system identifier.
+#. Remove the initialize key from DCS or wipe complete cluster state from DCS. The second one could be achieved by running ``patronictl remove <cluster-name>``. It is necessary because pg_upgrade runs initdb which actually creates a new database with a new PostgreSQL system identifier.
-5. If you wiped the cluster state in the previous step, you may wish to copy patroni.dynamic.json from old data dir to the new one.  It will help you to retain some PostgreSQL parameters you had set before.
+#. If you wiped the cluster state in the previous step, you may wish to copy patroni.dynamic.json from old data dir to the new one.  It will help you to retain some PostgreSQL parameters you had set before.
-6. Start Patroni on the primary node.
+#. Start Patroni on the primary node.
-7. Upgrade PostgreSQL binaries, update patroni.yml and wipe the data_dir on standby nodes.
+#. Upgrade PostgreSQL binaries, update patroni.yml and wipe the data_dir on standby nodes.
-8. Start Patroni on the standby nodes and wait for the replication to complete.
+#. Start Patroni on the standby nodes and wait for the replication to complete.
 
 Running pg_upgrade on standby nodes is not supported by PostgreSQL. If you know what you are doing, you can try the rsync procedure described in https://www.postgresql.org/docs/current/pgupgrade.html instead of wiping data_dir on standby nodes. The safest way is however to let Patroni replicate the data for you.
 

docs/ha_multi_dc.rst

@@ -12,7 +12,7 @@ In both cases, it is important to be clear about the following concepts:
 - You should run the odd number of etcd, ZooKeeper or Consul nodes: 3 or 5!
 
 Synchronous Replication
-----------------------------
+-----------------------
 
 To have a multi DC cluster that can automatically tolerate a zone drop, a minimum of 3 is required.
 
@@ -27,7 +27,7 @@ Regarding postgres, we must deploy at least 2 nodes, in different DC. Then you h
 This enables sync replication and the primary node will choose one of the nodes as synchronous.
 
 Asynchronous Replication
-----------------------------------
+------------------------
 
 With only two data centers it would be better to have two independent etcd clusters and run Patroni :ref:`standby cluster <standby_cluster>` in the second data center. If the first site is down, you can MANUALLY promote the ``standby_cluster``.
 

docs/index.rst

@@ -22,6 +22,7 @@ Currently supported PostgreSQL versions: 9.3 to 15.
    :caption: Contents:
 
    README
+   installation
    patroni_configuration
    rest_api
    replica_bootstrap
@@ -40,6 +41,13 @@ Currently supported PostgreSQL versions: 9.3 to 15.
 Indices and tables
 ==================
 
-* :ref:`genindex`
+.. ifconfig:: builder == 'html'
-* :ref:`modindex`
+
-* :ref:`search`
+  * :ref:`genindex`
+  * :ref:`modindex`
+  * :ref:`search`
+
+.. ifconfig:: builder != 'html'
+
+  * :ref:`genindex`
+  * :ref:`search`

docs/installation.rst

@@ -0,0 +1,201 @@
+.. _installation:
+
+Installation
+============
+
+Pre-requirements for Mac OS
+---------------------------
+
+To install requirements on a Mac, run the following:
+
+.. code-block:: shell
+
+    brew install postgresql etcd haproxy libyaml python
+
+.. _psycopg2_install_options:
+
+Psycopg
+-------
+
+Starting from `psycopg2-2.8`_ the binary version of psycopg2 will no longer be installed by default. Installing it from
+the source code requires C compiler and postgres+python dev packages. Since in the python world it is not possible to
+specify dependency as ``psycopg2 OR psycopg2-binary`` you will have to decide how to install it.
+
+There are a few options available:
+
+1. Use the package manager from your distro
+
+.. code-block:: shell
+
+    sudo apt-get install python3-psycopg2  # install psycopg2 module on Debian/Ubuntu
+    sudo yum install python3-psycopg2      # install psycopg2 on RedHat/Fedora/CentOS
+
+2. Install psycopg2 from the binary package
+
+.. code-block:: shell
+
+    pip install psycopg2-binary
+
+3. Install psycopg2 from source
+
+.. code-block:: shell
+
+    pip install psycopg2>=2.5.4
+
+4. Use psycopg 3.0 instead of psycopg2
+
+.. code-block:: shell
+
+    pip install psycopg[binary]>=3.0.0
+
+General installation for pip
+----------------------------
+
+Patroni can be installed with pip:
+
+.. code-block:: shell
+
+    pip install patroni[dependencies]
+
+where ``dependencies`` can be either empty, or consist of one or more of the following:
+
+etcd or etcd3
+    `python-etcd` module in order to use Etcd as Distributed Configuration Store (DCS)
+consul
+    `python-consul` module in order to use Consul as DCS
+zookeeper
+    `kazoo` module in order to use Zookeeper as DCS
+exhibitor
+    `kazoo` module in order to use Exhibitor as DCS (same dependencies as for Zookeeper)
+kubernetes
+    `kubernetes` module in order to use Kubernetes as DCS in Patroni
+raft
+    `pysyncobj` module in order to use python Raft implementation as DCS
+aws
+    `boto3` in order to use AWS callbacks
+
+For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is:
+
+.. code-block:: shell
+
+    pip install patroni[etcd,aws]
+
+Note that external tools to call in the replica creation or custom bootstrap scripts (i.e. WAL-E) should be installed
+independently of Patroni.
+
+.. _package_installation:
+
+Package installation on Linux
+-----------------------------
+
+Patroni packages may be available for your operating system, produced by the Postgres community for:
+
+* RHEL, RockyLinux, AlmaLinux;
+* Debian and Ubuntu;
+* SUSE Enterprise Linux.
+
+You can also find packages for direct dependencies of Patroni, like python modules that might not be available in
+the official operating system repositories.
+
+For more information see the `PGDG repository`_ documentation.
+
+If you are on a RedHat Enterprise Linux derivative operating system you may also require packages from EPEL, see
+`EPEL repository`_ documentation.
+
+Once you have installed the PGDG repository for your OS you can install patroni.
+
+.. note::
+
+    Patroni packages are not maintained by the Patroni developers, but rather by the Postgres community. If you
+    require support please first try connecting on `Postgres slack`_.
+
+Installing on Debian derivatives
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+With PGDG repo installed, see :ref:`above <package_installation>`, install Patroni via apt run:
+
+.. code-block:: shell
+
+    apt-get install patroni
+
+Installing on RedHat derivatives
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+With PGDG repo installed, see :ref:`above <package_installation>`, install patroni with an etcd DCS via dnf on RHEL 9
+(and derivatives) run:
+
+.. code-block:: shell
+
+    dnf install patroni patroni-etcd
+
+You can install etcd from PGDG if your RedHat derivative distribution does not provide packages. On the nodes that will
+host the DCS run:
+
+.. code-block:: shell
+
+    dnf install 'dnf-command(config-manager)'
+    dnf config-manager --enable pgdg-rhel9-extras
+    dnf install etcd
+
+You can replace the version of RHEL with `8` in the repo to make `pgdg-rhel8-extras` if needed. The repo name is still
+`pgdg-rhelN-extras` on RockyLinux, AlmaLinux, Oracle Linux, etc...
+
+Installing on SUSE Enterprise Linux
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You might need to enable the SUSE PackageHub repositories for some dependencies. see `SUSE PackageHub`_ documentation.
+
+For SLES 15 with PGDG repo installed, see :ref:`above <package_installation>`, you can install patroni using:
+
+.. code-block:: shell
+
+    zypper install patroni patroni-etcd
+
+With the SUSE PackageHub repo enabled you can also install etcd:
+
+.. code-block:: shell
+
+    SUSEConnect -p PackageHub/15.5/x86_64
+    zypper install etcd
+
+Upgrading
+---------
+
+Upgrading patroni is a very simple process, just update the software installation and restart the Patroni daemon on
+each node in the cluster.
+
+However, restarting the Patroni daemon will result in a Postgres database restart. In some situations this may cause
+a failover of the primary node in your cluster, therefore it is recommended to put the cluster into maintenance mode
+until the Patroni daemon restart has been completed.
+
+To put the cluster in maintenance mode, run the following command on one of the patroni nodes:
+
+.. code-block:: shell
+
+    patronictl pause --wait
+
+Then on each node in the cluster, perform the package upgrade required for your OS:
+
+.. code-block:: shell
+
+    apt-get update && apt-get install patroni patroni-etcd
+
+Restart the patroni daemon process on each node:
+
+.. code-block:: shell
+
+    systemctl restart patroni
+
+Then finally resume monitoring of Postgres with patroni to take it out of maintenance mode:
+
+.. code-block:: shell
+
+    patronictl resume --wait
+
+The cluster will now be full operational with the new version of Patroni.
+
+.. _psycopg2-2.8: http://initd.org/psycopg/articles/2019/04/04/psycopg-28-released/
+.. _PGDG repository: https://www.postgresql.org/download/linux/
+.. _EPEL repository: https://docs.fedoraproject.org/en-US/epel/
+.. _SUSE PackageHub: https://packagehub.suse.com/how-to-use/
+.. _Postgres slack: http://pgtreats.info/slack-invite

docs/kubernetes.rst

@@ -32,6 +32,58 @@ Configuration
 
 Patroni Kubernetes :ref:`settings <kubernetes_settings>` and :ref:`environment variables <kubernetes_environment>` are described in the general chapters of the documentation.
 
+.. _kubernetes_role_values:
+
+Customize role label
+^^^^^^^^^^^^^^^^^^^^
+
+By default, Patroni will set corresponding labels on the pod it runs in based on node's role, such as ``role=master``.
+The key and value of label can be customized by `kubernetes.role_label`, `kubernetes.leader_label_value`, `kubernetes.follower_label_value` and `kubernetes.standby_leader_label_value`.
+
+Note that if you migrate from default role labels to custom ones, you can reduce downtime by following migration steps:
+
+1. Add a temporary label using original role value for the pod with `kubernetes.tmp_role_label` (like ``tmp_role``). Once pods are restarted they will get following labels set by Patroni:
+
+  .. code:: YAML
+
+    labels:
+      cluster-name: foo
+      role: master
+      tmp_role: master
+
+2. After all pods have been updated, modify the service selector to select the temporary label.
+
+  .. code:: YAML
+
+    selector:
+      cluster-name: foo
+      tmp_role: master
+
+3. Add your custom role label (e.g., set `kubernetes.leader_label_value=primary`). Once pods are restarted they will get following new labels set by Patroni:
+
+  .. code:: YAML
+
+    labels:
+      cluster-name: foo
+      role: primary
+      tmp_role: master
+
+4. After all pods have been updated again, modify the service selector to use new role value.
+
+  .. code:: YAML
+
+    selector:
+      cluster-name: foo
+      role: primary
+
+5. Finally, remove the temporary label from your configuration and update all pods.
+
+  .. code:: YAML
+
+    labels:
+      cluster-name: foo
+      role: primary
+
 Examples
 --------
 

docs/patroni_configuration.rst

@@ -44,7 +44,6 @@ Some of the PostgreSQL parameters **must hold the same values on the primary and
 - **max_worker_processes**: 8
 - **max_prepared_transactions**: 0
 - **wal_level**: hot_standby
-- **wal_log_hints**: on
 - **track_commit_timestamp**: off
 
 For the parameters below, PostgreSQL does not require equal values among the primary and all the replicas. However, considering the possibility of a replica to become the primary at any time, it doesn't really make sense to set them differently; therefore, **Patroni restricts setting their values to the** :ref:`dynamic configuration <dynamic_configuration>`.
@@ -62,6 +61,7 @@ There are some other Postgres parameters controlled by Patroni:
 - **port** - is set either from ``postgresql.listen`` or from ``PATRONI_POSTGRESQL_LISTEN`` environment variable
 - **cluster_name** - is set either from ``scope`` or from ``PATRONI_SCOPE`` environment variable
 - **hot_standby: on**
+- **wal_log_hints: on** - for Postgres 9.4 and newer.
 
 To be on the safe side parameters from the above lists are not written into ``postgresql.conf``, but passed as a list of arguments to the ``pg_ctl start`` which gives them the highest precedence, even above `ALTER SYSTEM <https://www.postgresql.org/docs/current/static/sql-altersystem.html>`__
 
@@ -90,6 +90,42 @@ The parameters would be applied in the following order (run-time are given the h
 This allows configuration for all the nodes (2), configuration for a specific node using ``ALTER SYSTEM`` (3) and ensures that parameters essential to the running of Patroni are enforced (4), as well as leaves room for configuration tools that manage `postgresql.conf` directly without involving Patroni (1).
 
 
+PostgreSQL parameters that touch shared memory
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+PostgreSQL has some parameters that determine the size of the shared memory used by them:
+
+- **max_connections**
+- **max_prepared_transactions**
+- **max_locks_per_transaction**
+- **max_wal_senders**
+- **max_worker_processes**
+
+Changing these parameters require a PostgreSQL restart to take effect, and their shared memory structures cannot be smaller on the standby nodes than on the primary node.
+
+As explained before, Patroni restrict changing their values through :ref:`dynamic configuration <dynamic_configuration>`, which usually consists of:
+
+1. Applying changes through ``patronictl edit-config`` (or via REST API ``/config`` endpoint)
+2. Restarting nodes through ``patronictl restart`` (or via REST API ``/restart`` endpoint)
+
+**Note:** please keep in mind that you should perform a restart of the PostgreSQL nodes through ``patronictl restart`` command, or via REST API ``/restart`` endpoint. An attempt to restart PostgreSQL by restarting the Patroni daemon, e.g. by executing ``systemctl restart patroni``, can cause a failover to occur in the cluster, if you are restarting the primary node.
+
+However, as those settings manage shared memory, some extra care should be taken when restarting the nodes:
+
+* If you want to **increase** the value of any of those settings:
+
+   1. Restart all standbys first
+   2. Restart the primary after that
+
+* If you want to **decrease** the value of any of those settings:
+
+   1. Restart the primary first
+   2. Restart all standbys after that
+
+**Note:** if you attempt to restart all nodes in one go after **decreasing** the value of any of those settings, Patroni will ignore the change and restart the standby with the original setting value, thus requiring that you restart the standbys again later. Patroni does that to prevent the standby to enter in an infinite crash loop, because PostgreSQL quits with a `FATAL` message if you attempt to set any of those parameters to a value lower than what is visible in ``pg_controldata`` on the Standby node. In other words, we can only decrease the setting on the standby once its ``pg_controldata`` is up-to-date with the primary in regards to these changes on the primary.
+
+More information about that can be found at `PostgreSQL Administrator's Overview <https://www.postgresql.org/docs/current/hot-standby.html#HOT-STANDBY-ADMIN>`__.
+
 Patroni configuration parameters
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

docs/pause.rst

@@ -19,7 +19,7 @@ When Patroni runs in a paused mode, it does not change the state of PostgreSQL,
 
 - For the Postgres primary with the leader lock Patroni updates the lock. If the node with the leader lock stops being the primary (i.e. is demoted manually), Patroni will release the lock instead of promoting the node back.
 
-- Manual unscheduled restart, reinitialize and manual failover are allowed. Manual failover is only allowed if the node to failover to is specified. In the paused mode, manual failover does not require a running primary node.
+- Manual unscheduled restart, manual unscheduled failover/switchover and reinitialize are allowed. No scheduled action is allowed. Manual switchover is only allowed if the node to switch over to is specified.
 
 - If 'parallel' primaries are detected by Patroni, it emits a warning, but does not demote the primary without the leader lock.
 

docs/releases.rst

@@ -3,6 +3,84 @@
 Release notes
 =============
 
+Version 3.1.0
+-------------
+
+**Breaking changes**
+
+- Changed semantic of ``restapi.keyfile`` and ``restapi.certfile`` (Alexander Kukushkin)
+
+  Previously Patroni was using ``restapi.keyfile`` and ``restapi.certfile`` as client certificates as a fallback if there were no respective configuration parameters in the ``ctl`` section.
+
+.. warning::
+    If you enabled client certificates validation (``restapi.verify_client`` is set to ``required``), you also **must** provide **valid client certificates** in the ``ctl.certfile``, ``ctl.keyfile``, ``ctl.keyfile_password``. If not provided, Patroni will not work correctly.
+
+
+**New features**
+
+- Make Pod role label configurable (Waynerv)
+
+  Values could be customized using ``kubernetes.leader_label_value``, ``kubernetes.follower_label_value`` and ``kubernetes.standby_leader_label_value`` parameters. This feature will be very useful when we change the ``master`` role to the ``primary``. You can read more about the feature and migration steps :ref:`here <kubernetes_role_values>`.
+
+
+**Improvements**
+
+- Various improvements of ``patroni --validate-config`` (Alexander Kukushkin)
+
+  Improved parameter validation for different DCS, ``bootstrap.dcs`` , ``ctl``, ``restapi``, and ``watchdog`` sections.
+
+- Start Postgres not in recovery if it crashed during recovery while Patroni is running (Alexander Kukushkin)
+
+  It may reduce recovery time and will help to prevent unnecessary timeline increments.
+
+- Avoid unnecessary updates of ``/status`` key (Alexander Kukushkin)
+
+  When there are no permanent logical slots Patroni was updating the ``/status`` on every heartbeat loop even when LSN on the primary didn't move forward.
+
+- Don't allow stale primary to win the leader race (Alexander Kukushkin)
+
+  If Patroni was hanging during a significant time due to lack of resources it will additionally check that no other nodes promoted Postgres before acquiring the leader lock.
+
+- Implemented visibility of certain PostgreSQL parameters validation (Alexander Kukushkin, Feike Steenbergen)
+
+  If validation of ``max_connections``, ``max_wal_senders``, ``max_prepared_transactions``, ``max_locks_per_transaction``, ``max_replication_slots``, or ``max_worker_processes`` failed Patroni was using some sane default value. Now in addition to that it will also show a warning.
+
+- Set permissions for files and directories created in ``PGDATA`` (Alexander Kukushkin)
+
+  All files created by Patroni had only owner read/write permissions. This behaviour was breaking backup tools that run under a different user and relying on group read permissions. Now Patroni honors permissions on ``PGDATA`` and correctly sets permissions on all directories and files it creates inside ``PGDATA``.
+
+
+**Bugfixes**
+
+- Run ``archive_command`` through shell (Waynerv)
+
+  Patroni might archive some WAL segments before doing crash recovery in a single-user mode or before ``pg_rewind``. If the archive_command contains some shell operators, like ``&&`` it didn't work with Patroni.
+
+- Fixed "on switchover" shutdown checks (Polina Bungina)
+
+  It was possible that specified candidate is still streaming and didn't received shut down checking but the leader key was removed because some other nodes were healthy.
+
+- Fixed "is primary" check (Alexander Kukushkin)
+
+  During the leader race replicas were not able to recognize that Postgres on the old leader is still running as a primary.
+
+- Fixed ``patronictl list`` (Alexander Kukushkin)
+
+  The Cluster name field was missing in ``tsv``, ``json``, and ``yaml`` output formats.
+
+- Fixed ``pg_rewind`` behaviour after pause (Alexander Kukushkin)
+
+  Under certain conditions, Patroni wasn't able to join the false primary back to the cluster with ``pg_rewind`` after coming out of maintenance mode.
+
+- Fixed bug in Etcd v3 implementation (Alexander Kukushkin)
+
+  Invalidate internal KV cache if key update performed using ``create_revision``/``mod_revision`` field due to revision mismatch.
+
+- Fixed behaviour of replicas in standby cluster in pause (Alexander Kukushkin)
+
+  When the leader key expires replicas in standby cluster will not follow the remote node but keep ``primary_conninfo`` as it is.
+
+
 Version 3.0.4
 -------------
 

docs/rest_api.rst

@@ -92,26 +92,188 @@ Monitoring endpoint
 
 The ``GET /patroni`` is used by Patroni during the leader race. It also could be used by your monitoring system. The JSON document produced by this endpoint has the same structure as the JSON produced by the health check endpoints.
 
+**Example:** A healthy cluster
+
 .. code-block:: bash
 
     $ curl -s http://localhost:8008/patroni | jq .
     {
       "state": "running",
-      "postmaster_start_time": "2019-09-24 09:22:32.555 CEST",
+      "postmaster_start_time": "2023-08-18 11:03:37.966359+00:00",
       "role": "master",
-      "server_version": 110005,
+      "server_version": 150004,
-      "cluster_unlocked": false,
       "xlog": {
-        "location": 25624640
+        "location": 67395656
       },
-      "timeline": 3,
+      "timeline": 1,
-      "database_system_identifier": "6739877027151648096",
+      "replication": [
+        {
+          "usename": "replicator",
+          "application_name": "patroni2",
+          "client_addr": "10.89.0.6",
+          "state": "streaming",
+          "sync_state": "async",
+          "sync_priority": 0
+        },
+        {
+          "usename": "replicator",
+          "application_name": "patroni3",
+          "client_addr": "10.89.0.2",
+          "state": "streaming",
+          "sync_state": "async",
+          "sync_priority": 0
+        }
+      ],
+      "dcs_last_seen": 1692356718,
+      "tags": {
+        "clonefrom": true
+      },
+      "database_system_identifier": "7268616322854375442",
       "patroni": {
-        "version": "1.6.0",
+        "version": "3.1.0",
-        "scope": "batman"
+        "scope": "demo",
+        "name": "patroni1"
       }
     }
 
+**Example:** An unlocked cluster
+
+.. code-block:: bash
+
+    $ curl -s http://localhost:8008/patroni  | jq .
+    {
+      "state": "running",
+      "postmaster_start_time": "2023-08-18 11:09:08.615242+00:00",
+      "role": "replica",
+      "server_version": 150004,
+      "xlog": {
+        "received_location": 67419744,
+        "replayed_location": 67419744,
+        "replayed_timestamp": null,
+        "paused": false
+      },
+      "timeline": 1,
+      "replication": [
+        {
+          "usename": "replicator",
+          "application_name": "patroni2",
+          "client_addr": "10.89.0.6",
+          "state": "streaming",
+          "sync_state": "async",
+          "sync_priority": 0
+        },
+        {
+          "usename": "replicator",
+          "application_name": "patroni3",
+          "client_addr": "10.89.0.2",
+          "state": "streaming",
+          "sync_state": "async",
+          "sync_priority": 0
+        }
+      ],
+      "cluster_unlocked": true,
+      "dcs_last_seen": 1692356928,
+      "tags": {
+        "clonefrom": true
+      },
+      "database_system_identifier": "7268616322854375442",
+      "patroni": {
+        "version": "3.1.0",
+        "scope": "demo",
+        "name": "patroni1"
+      }
+    }
+
+**Example:** An unlocked cluster with :ref:`DCS failsafe mode <dcs_failsafe_mode>` enabled
+
+.. code-block:: bash
+
+    $ curl -s http://localhost:8008/patroni  | jq .
+    {
+      "state": "running",
+      "postmaster_start_time": "2023-08-18 11:09:08.615242+00:00",
+      "role": "replica",
+      "server_version": 150004,
+      "xlog": {
+        "location": 67420024
+      },
+      "timeline": 1,
+      "replication": [
+        {
+          "usename": "replicator",
+          "application_name": "patroni2",
+          "client_addr": "10.89.0.6",
+          "state": "streaming",
+          "sync_state": "async",
+          "sync_priority": 0
+        },
+        {
+          "usename": "replicator",
+          "application_name": "patroni3",
+          "client_addr": "10.89.0.2",
+          "state": "streaming",
+          "sync_state": "async",
+          "sync_priority": 0
+        }
+      ],
+      "cluster_unlocked": true,
+      "failsafe_mode_is_active": true,
+      "dcs_last_seen": 1692356928,
+      "tags": {
+        "clonefrom": true
+      },
+      "database_system_identifier": "7268616322854375442",
+      "patroni": {
+        "version": "3.1.0",
+        "scope": "demo",
+        "name": "patroni1"
+      }
+    }
+
+**Example:** A cluster with the :ref:`pause mode <pause>` enabled
+
+.. code-block:: bash
+
+    $ curl -s http://localhost:8008/patroni  | jq .
+    {
+      "state": "running",
+      "postmaster_start_time": "2023-08-18 11:09:08.615242+00:00",
+      "role": "replica",
+      "server_version": 150004,
+      "xlog": {
+        "location": 67420024
+      },
+      "timeline": 1,
+      "replication": [
+        {
+          "usename": "replicator",
+          "application_name": "patroni2",
+          "client_addr": "10.89.0.6",
+          "state": "streaming",
+          "sync_state": "async",
+          "sync_priority": 0
+        },
+        {
+          "usename": "replicator",
+          "application_name": "patroni3",
+          "client_addr": "10.89.0.2",
+          "state": "streaming",
+          "sync_state": "async",
+          "sync_priority": 0
+        }
+      ],
+      "pause": true,
+      "dcs_last_seen": 1692356928,
+      "tags": {
+        "clonefrom": true
+      },
+      "database_system_identifier": "7268616322854375442",
+      "patroni": {
+        "version": "3.1.0",
+        "scope": "demo",
+        "name": "patroni1"
+      }
+    }
 
 Retrieve the Patroni metrics in Prometheus format through the ``GET /metrics`` endpoint.
 
@@ -121,64 +283,70 @@ Retrieve the Patroni metrics in Prometheus format through the ``GET /metrics`` e
 	
 	# HELP patroni_version Patroni semver without periods. \
 	# TYPE patroni_version gauge
-	patroni_version{scope="batman"} 020103
+	patroni_version{scope="batman",name="patroni1"} 020103
 	# HELP patroni_postgres_running Value is 1 if Postgres is running, 0 otherwise.
 	# TYPE patroni_postgres_running gauge
-	patroni_postgres_running{scope="batman"} 1
+	patroni_postgres_running{scope="batman",name="patroni1"} 1
 	# HELP patroni_postmaster_start_time Epoch seconds since Postgres started.
 	# TYPE patroni_postmaster_start_time gauge
-	patroni_postmaster_start_time{scope="batman"} 1657656955.179243
+	patroni_postmaster_start_time{scope="batman",name="patroni1"} 1657656955.179243
 	# HELP patroni_master Value is 1 if this node is the leader, 0 otherwise.
 	# TYPE patroni_master gauge
-	patroni_master{scope="batman"} 1
+	patroni_master{scope="batman",name="patroni1"} 1
+	# HELP patroni_primary Value is 1 if this node is the leader, 0 otherwise.
+	# TYPE patroni_primary gauge
+	patroni_primary{scope="batman",name="patroni1"} 1
 	# HELP patroni_xlog_location Current location of the Postgres transaction log, 0 if this node is not the leader.
 	# TYPE patroni_xlog_location counter
-	patroni_xlog_location{scope="batman"} 22320573386952
+	patroni_xlog_location{scope="batman",name="patroni1"} 22320573386952
 	# HELP patroni_standby_leader Value is 1 if this node is the standby_leader, 0 otherwise.
 	# TYPE patroni_standby_leader gauge
-	patroni_standby_leader{scope="batman"} 0
+	patroni_standby_leader{scope="batman",name="patroni1"} 0
 	# HELP patroni_replica Value is 1 if this node is a replica, 0 otherwise.
 	# TYPE patroni_replica gauge
-	patroni_replica{scope="batman"} 0
+	patroni_replica{scope="batman",name="patroni1"} 0
 	# HELP patroni_sync_standby Value is 1 if this node is a sync standby replica, 0 otherwise.
 	# TYPE patroni_sync_standby gauge
-	patroni_sync_standby{scope="batman"} 0
+	patroni_sync_standby{scope="batman",name="patroni1"} 0
 	# HELP patroni_xlog_received_location Current location of the received Postgres transaction log, 0 if this node is not a replica.
 	# TYPE patroni_xlog_received_location counter
-	patroni_xlog_received_location{scope="batman"} 0
+	patroni_xlog_received_location{scope="batman",name="patroni1"} 0
 	# HELP patroni_xlog_replayed_location Current location of the replayed Postgres transaction log, 0 if this node is not a replica.
 	# TYPE patroni_xlog_replayed_location counter
-	patroni_xlog_replayed_location{scope="batman"} 0
+	patroni_xlog_replayed_location{scope="batman",name="patroni1"} 0
 	# HELP patroni_xlog_replayed_timestamp Current timestamp of the replayed Postgres transaction log, 0 if null.
 	# TYPE patroni_xlog_replayed_timestamp gauge
-	patroni_xlog_replayed_timestamp{scope="batman"} 0
+	patroni_xlog_replayed_timestamp{scope="batman",name="patroni1"} 0
 	# HELP patroni_xlog_paused Value is 1 if the Postgres xlog is paused, 0 otherwise.
 	# TYPE patroni_xlog_paused gauge
-	patroni_xlog_paused{scope="batman"} 0
+	patroni_xlog_paused{scope="batman",name="patroni1"} 0
 	# HELP patroni_postgres_streaming Value is 1 if Postgres is streaming, 0 otherwise.
 	# TYPE patroni_postgres_streaming gauge
-	patroni_postgres_streaming{scope="batman"} 1
+	patroni_postgres_streaming{scope="batman",name="patroni1"} 1
 	# HELP patroni_postgres_in_archive_recovery Value is 1 if Postgres is replicating from archive, 0 otherwise.
 	# TYPE patroni_postgres_in_archive_recovery gauge
-	patroni_postgres_in_archive_recovery{scope="batman"} 0
+	patroni_postgres_in_archive_recovery{scope="batman",name="patroni1"} 0
 	# HELP patroni_postgres_server_version Version of Postgres (if running), 0 otherwise.
 	# TYPE patroni_postgres_server_version gauge
-	patroni_postgres_server_version {scope="batman"} 140004
+	patroni_postgres_server_version{scope="batman",name="patroni1"} 140004
 	# HELP patroni_cluster_unlocked Value is 1 if the cluster is unlocked, 0 if locked.
 	# TYPE patroni_cluster_unlocked gauge
-	patroni_cluster_unlocked{scope="batman"} 0
+	patroni_cluster_unlocked{scope="batman",name="patroni1"} 0
 	# HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise.
 	# TYPE patroni_postgres_timeline counter
-	patroni_postgres_timeline{scope="batman"} 24
+	patroni_failsafe_mode_is_active{scope="batman",name="patroni1"} 0
+	# HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise.
+	# TYPE patroni_postgres_timeline counter
+	patroni_postgres_timeline{scope="batman",name="patroni1"} 24
 	# HELP patroni_dcs_last_seen Epoch timestamp when DCS was last contacted successfully by Patroni.
 	# TYPE patroni_dcs_last_seen gauge
-	patroni_dcs_last_seen{scope="batman"} 1677658321
+	patroni_dcs_last_seen{scope="batman",name="patroni1"} 1677658321
 	# HELP patroni_pending_restart Value is 1 if the node needs a restart, 0 otherwise.
 	# TYPE patroni_pending_restart gauge
-	patroni_pending_restart{scope="batman"} 1
+	patroni_pending_restart{scope="batman",name="patroni1"} 1
 	# HELP patroni_is_paused Value is 1 if auto failover is disabled, 0 otherwise.
 	# TYPE patroni_is_paused gauge
-	patroni_is_paused{scope="batman"} 1
+	patroni_is_paused{scope="batman",name="patroni1"} 1
 
 
 Cluster status endpoints
@@ -192,24 +360,24 @@ Cluster status endpoints
     {
       "members": [
         {
-          "name": "postgresql0",
+          "name": "patroni1",
-          "host": "127.0.0.1",
-          "port": 5432,
           "role": "leader",
           "state": "running",
-          "api_url": "http://127.0.0.1:8008/patroni",
+          "api_url": "http://10.89.0.4:8008/patroni",
+          "host": "10.89.0.4",
+          "port": 5432,
           "timeline": 5,
           "tags": {
             "clonefrom": true
           }
         },
         {
-          "name": "postgresql1",
+          "name": "patroni2",
-          "host": "127.0.0.1",
-          "port": 5433,
           "role": "replica",
-          "state": "running",
+          "state": "streaming",
-          "api_url": "http://127.0.0.1:8009/patroni",
+          "api_url": "http://10.89.0.6:8008/patroni",
+          "host": "10.89.0.6",
+          "port": 5433,
           "timeline": 5,
           "tags": {
             "clonefrom": true
@@ -217,9 +385,11 @@ Cluster status endpoints
           "lag": 0
         }
       ],
+      "scope": "demo",
       "scheduled_switchover": {
-        "at": "2019-09-24T10:36:00+02:00",
+        "at": "2023-09-24T10:36:00+02:00",
-        "from": "postgresql0"
+        "from": "patroni1",
+        "to": "patroni3"
       }
     }
 
@@ -264,7 +434,7 @@ Config endpoint
 
 .. code-block:: bash
 
-	$ curl -s localhost:8008/config | jq .
+	$ curl -s http://localhost:8008/config | jq .
 	{
 	  "ttl": 30,
 	  "loop_wait": 10,
@@ -275,7 +445,6 @@ Config endpoint
 	    "use_pg_rewind": true,
 	    "parameters": {
 	      "hot_standby": "on",
-	      "wal_log_hints": "on",
 	      "wal_level": "hot_standby",
 	      "max_wal_senders": 5,
 	      "max_replication_slots": 5,
@@ -302,7 +471,6 @@ Config endpoint
 	    "use_pg_rewind": true,
 	    "parameters": {
 	      "hot_standby": "on",
-	      "wal_log_hints": "on",
 	      "wal_level": "hot_standby",
 	      "max_wal_senders": 5,
 	      "max_replication_slots": 5,
@@ -326,8 +494,9 @@ Let's check that the node processed this configuration. First of all it should s
 	    "location": 2197818976
 	  },
 	  "patroni": {
+	    "version": "1.0",
 	    "scope": "batman",
-	    "version": "1.0"
+	    "name": "patroni1"
 	  },
 	  "state": "running",
 	  "role": "master",
@@ -355,7 +524,6 @@ If you want to remove (reset) some setting just patch it with ``null``:
 	      "hot_standby": "on",
 	      "unix_socket_directories": ".",
 	      "wal_level": "hot_standby",
-	      "wal_log_hints": "on",
 	      "max_wal_senders": 5,
 	      "max_replication_slots": 5
 	    }
@@ -369,7 +537,7 @@ The above call removes ``postgresql.parameters.max_connections`` from the dynami
 .. code-block:: bash
 
 	$ curl -s -XPUT -d \
-		'{"maximum_lag_on_failover":1048576,"retry_timeout":10,"postgresql":{"use_slots":true,"use_pg_rewind":true,"parameters":{"hot_standby":"on","wal_log_hints":"on","wal_level":"hot_standby","unix_socket_directories":".","max_wal_senders":5}},"loop_wait":3,"ttl":20}' \
+		'{"maximum_lag_on_failover":1048576,"retry_timeout":10,"postgresql":{"use_slots":true,"use_pg_rewind":true,"parameters":{"hot_standby":"on","wal_level":"hot_standby","unix_socket_directories":".","max_wal_senders":5}},"loop_wait":3,"ttl":20}' \
 		http://localhost:8008/config | jq .
 	{
 	  "ttl": 20,
@@ -381,7 +549,6 @@ The above call removes ``postgresql.parameters.max_connections`` from the dynami
 	      "hot_standby": "on",
 	      "unix_socket_directories": ".",
 	      "wal_level": "hot_standby",
-	      "wal_log_hints": "on",
 	      "max_wal_senders": 5
 	    },
 	    "use_pg_rewind": true
@@ -393,39 +560,111 @@ The above call removes ``postgresql.parameters.max_connections`` from the dynami
 Switchover and failover endpoints
 ---------------------------------
 
-``POST /switchover`` or ``POST /failover``. These endpoints are very similar to each other. There are a couple of minor differences though:
+.. _switchover_api:
 
-1. The failover endpoint allows to perform a manual failover when there are no healthy nodes, but at the same time it will not allow you to schedule a switchover.
+Switchover
+^^^^^^^^^^
 
-2. The switchover endpoint is the opposite. It works only when the cluster is healthy (there is a leader) and allows to schedule a switchover at a given time.
+``/switchover`` endpoint only works when the cluster is healthy (there is a leader). It also allows to schedule a switchover at a given time.
 
+When calling ``/switchover`` endpoint a candidate can be specified but is not required, in contrast to ``/failover`` endpoint. If a candidate is not provided, all the eligible nodes of the cluster will participate in the leader race after the leader stepped down.
 
-In the JSON body of the ``POST`` request you must specify at least the ``leader`` or ``candidate`` fields and optionally the ``scheduled_at`` field if you want to schedule a switchover at a specific time.
+In the JSON body of the ``POST`` request you must specify the ``leader`` field. The ``candidate`` and the ``scheduled_at`` fields are optional and can be used to schedule a switchover at a specific time.
 
+Depending on the situation, requests might return different HTTP status codes and bodies. Status code **200** is returned when the switchover or failover successfully completed. If the switchover was successfully scheduled, Patroni will return HTTP status code **202**. In case something went wrong, the error status code (one of **400**, **412**, or **503**) will be returned with some details in the response body.
 
-Example: perform a failover to the specific node:
+``DELETE /switchover`` can be used to delete the currently scheduled switchover.
+
+**Example:** perform a switchover to any healthy standby
 
 .. code-block:: bash
 
-    $ curl -s http://localhost:8009/failover -XPOST -d '{"candidate":"postgresql1"}'
+	$ curl -s http://localhost:8008/switchover -XPOST -d '{"leader":"postgresql1"}'
-    Successfully failed over to "postgresql1"
+	Successfully switched over to "postgresql2"
 
 
-Example: schedule a switchover from the leader to any other healthy replica in the cluster at a specific time:
+**Example:** perform a switchover to a specific node
 
 .. code-block:: bash
 
-    $ curl -s http://localhost:8008/switchover -XPOST -d \
+	$ curl -s http://localhost:8008/switchover -XPOST -d \
-	    '{"leader":"postgresql0","scheduled_at":"2019-09-24T12:00+00"}'
+		'{"leader":"postgresql1","candidate":"postgresql2"}'
-    Switchover scheduled
+	Successfully switched over to "postgresql2"
 
 
-Depending on the situation the request might finish with a different HTTP status code and body. The status code **200** is returned when the switchover or failover successfully completed. If the switchover was successfully scheduled, Patroni will return HTTP status code **202**. In case something went wrong, the error status code (one of **400**, **412** or **503**) will be returned with some details in the response body. For more information please check the source code of ``patroni/api.py:do_POST_failover()`` method.
+**Example:** schedule a switchover from the leader to any other healthy standby in the cluster at a specific time.
 
-- ``DELETE /switchover``: delete the scheduled switchover
+.. code-block:: bash
 
-The ``POST /switchover`` and ``POST failover`` endpoints are used by ``patronictl switchover`` and ``patronictl failover``, respectively.
+	$ curl -s http://localhost:8008/switchover -XPOST -d \
-The ``DELETE /switchover`` is used by ``patronictl flush <cluster-name> switchover``.
+		'{"leader":"postgresql0","scheduled_at":"2019-09-24T12:00+00"}'
+	Switchover scheduled
+
+
+Failover
+^^^^^^^^
+
+``/failover`` endpoint can be used to perform a manual failover when there are no healthy nodes (e.g. to an asynchronous standby if all synchronous standbys are not healthy enough to promote). However there is no requirement for a cluster not to have leader - failover can also be run on a healthy cluster.
+
+In the JSON body of the ``POST`` request you must specify the ``candidate`` field. If the ``leader`` field is specified, a switchover is triggered instead.
+
+**Example:**
+
+.. code-block:: bash
+
+	$ curl -s http://localhost:8008/failover -XPOST -d '{"candidate":"postgresql1"}'
+	Successfully failed over to "postgresql1"
+
+.. warning::
+	:ref:`Be very careful <failover_healthcheck>` when using this endpoint, as this can cause data loss in certain situations. In most cases, :ref:`the switchover endpoint <switchover_api>` satisfies the administrator's needs. 
+
+
+``POST /switchover`` and ``POST /failover`` endpoints are used by ``patronictl switchover`` and ``patronictl failover``, respectively.
+
+``DELETE /switchover`` is used by ``patronictl flush <cluster-name> switchover``.
+
+.. list-table:: Failover/Switchover comparison
+   :widths: 25 25 25
+   :header-rows: 1
+
+   * -
+     - Failover
+     - Switchover
+   * - Requires leader specified
+     - no
+     - yes
+   * - Requires candidate specified
+     - yes
+     - no
+   * - Can be run in pause
+     - yes
+     - yes (only to a specific candidate)
+   * - Can be scheduled
+     - no
+     - yes (if not in pause)
+
+.. _failover_healthcheck:
+
+Healthy standby
+^^^^^^^^^^^^^^^
+
+There are a couple of checks that a member of a cluster should pass to be able to participate in the leader race during a switchover or to become a leader as a failover/switchover candidate:
+
+- be reachable via Patroni API;
+- not have ``nofailover`` tag set to ``true``;
+- have watchdog fully functional (if required by the configuration);
+- in case of a switchover in a healthy cluster or an automatic failover, not exceed maximum replication lag (``maximum_lag_on_failover`` :ref:`configuration parameter <dynamic_configuration>`);
+- in case of a switchover in a healthy cluster or an automatic failover, not have a timeline number smaller than the cluster timeline if ``check_timeline`` :ref:`configuration parameter <dynamic_configuration>` is set to ``true``;
+- in :ref:`synchronous mode <synchronous_mode>`:
+
+  - In case of a switchover (both with and without a candidate): be listed in the ``/sync`` key members;
+  - For a failover in both healthy and unhealthy clusters, this check is omitted.
+
+.. warning::
+    In case of a manual failover in a cluster without a leader, a candidate will be allowed to promote even if:
+	- it is not in the ``/sync`` key members when synchronous mode is enabled;
+	- its lag exceeds the maximum replication lag allowed;
+	- it has the timeline number smaller than the last known cluster timeline.
 
 
 Restart endpoint

docs/yaml_configuration.rst

@@ -30,9 +30,15 @@ Log
 
 Bootstrap configuration
 -----------------------
+
+.. note::
+    Once Patroni has initialized the cluster for the first time and settings have been stored in the DCS, all future
+    changes to the ``bootstrap.dcs`` section of the YAML configuration will not take any effect! If you want to change
+    them please use either ``patronictl edit-config`` or the Patroni :ref:`REST API <rest_api>`.
+
 -  **bootstrap**:
 
-   -  **dcs**: This section will be written into `/<namespace>/<scope>/config` of the given configuration store after initializing of new cluster. The global dynamic configuration for the cluster. Under the ``bootstrap.dcs`` you can put any of the parameters described in the :ref:`Dynamic Configuration settings <dynamic_configuration>` and after Patroni initialized (bootstrapped) the new cluster, it will write this section into `/<namespace>/<scope>/config` of the configuration store. All later changes of ``bootstrap.dcs`` will not take any effect! If you want to change them please use either ``patronictl edit-config`` or Patroni :ref:`REST API <rest_api>`.
+   -  **dcs**: This section will be written into `/<namespace>/<scope>/config` of the given configuration store after initializing the new cluster. The global dynamic configuration for the cluster. You can put any of the parameters described in the :ref:`Dynamic Configuration settings <dynamic_configuration>` under ``bootstrap.dcs`` and after Patroni has initialized (bootstrapped) the new cluster, it will write this section into `/<namespace>/<scope>/config` of the configuration store.
    -  **method**: custom script to use for bootstrapping this cluster.
 
       See :ref:`custom bootstrap methods documentation <custom_bootstrap>` for details.
@@ -43,17 +49,24 @@ Bootstrap configuration
       -  **- data-checksums**: Must be enabled when pg_rewind is needed on 9.3.
       -  **- encoding: UTF8**: default encoding for new databases.
       -  **- locale: UTF8**: default locale for new databases.
-   -  **users**: Some additional users which need to be created after initializing new cluster
+   -  **users**: Some additional users which need to be created after initializing new cluster, see :ref:`Bootstrap users configuration <bootstrap_users_configuration>` below.
-
-      -  **admin**: the name of user
-
-         -  **password**: (optional) password for the user
-         -  **options**: list of options for CREATE USER statement
-
-            -  **- createrole**
-            -  **- createdb**
    -  **post\_bootstrap** or **post\_init**: An additional script that will be executed after initializing the cluster. The script receives a connection string URL (with the cluster superuser as a user name). The PGPASSFILE variable is set to the location of pgpass file.
 
+.. _bootstrap_users_configuration:
+
+Bootstrap users configuration
+=============================
+
+Users which need to be created after initializing the cluster:
+
+-  **admin**: the name of user
+
+  -  **password**: (optional) password for the user
+  -  **options**: list of options for CREATE USER statement
+
+    -  **- createrole**
+    -  **- createdb**
+
 .. _citus_settings:
 
 Citus
@@ -155,7 +168,11 @@ Kubernetes
 -  **namespace**: (optional) Kubernetes namespace where Patroni pod is running. Default value is `default`.
 -  **labels**: Labels in format ``{label1: value1, label2: value2}``. These labels will be used to find existing objects (Pods and either Endpoints or ConfigMaps) associated with the current cluster. Also Patroni will set them on every object (Endpoint or ConfigMap) it creates.
 -  **scope\_label**: (optional) name of the label containing cluster name. Default value is `cluster-name`.
--  **role\_label**: (optional) name of the label containing role (master or replica). Patroni will set this label on the pod it runs in. Default value is ``role``.
+-  **role\_label**: (optional) name of the label containing role (master or replica or other custom value). Patroni will set this label on the pod it runs in. Default value is ``role``.
+-  **leader\_label\_value**: (optional) value of the pod label when Postgres role is ``master``. Default value is ``master``.
+-  **follower\_label\_value**: (optional) value of the pod label when Postgres role is ``replica``. Default value is ``replica``.
+-  **standby\_leader\_label\_value**: (optional) value of the pod label when Postgres role is ``standby_leader``. Default value is ``master``.
+-  **tmp_\role\_label**: (optional) name of the temporary label containing role (master or replica). Value of this label will always use the default of corresponding role. Set only when necessary.
 -  **use\_endpoints**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
 -  **pod\_ip**: (optional) IP address of the pod Patroni is running in. This value is required when `use_endpoints` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
 -  **ports**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``kubernetes.ports: [{"name": "postgresql", "port": 5432}]`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `kubernetes.use_endpoints` is set.
@@ -335,17 +352,27 @@ Here is an example of both **http_extra_headers** and **https_extra_headers**:
           https_extra_headers:
             'Strict-Transport-Security': 'max-age=31536000; includeSubDomains'
 
+.. warning::
+
+    - The ``restapi.connect_address`` must be accessible from all nodes of a given Patroni cluster. Internally Patroni is using it during the leader race to find nodes with minimal replication lag.
+    - If you enabled client certificates validation (``restapi.verify_client`` is set to ``required``), you also **must** provide **valid client certificates** in the ``ctl.certfile``, ``ctl.keyfile``, ``ctl.keyfile_password``. If not provided, Patroni will not work correctly.
+
+
 .. _patronictl_settings:
 
 CTL
 ---
 -  **ctl**: (optional)
 
+   -  **authentication**:
+
+      -  **username**: Basic-auth username for accessing protected REST API endpoints. If not provided patronictl will use the value provided for REST API "username" parameter.
+      -  **password**: Basic-auth password for accessing protected REST API endpoints. If not provided patronictl will use the value provided for REST API "password" parameter.
    -  **insecure**: Allow connections to REST API without verifying SSL certs.
    -  **cacert**: Specifies the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs. If not provided patronictl will use the value provided for REST API "cafile" parameter.
-   -  **certfile**: Specifies the file with the client certificate in the PEM format. If not provided patronictl will use the value provided for REST API "certfile" parameter.
+   -  **certfile**: Specifies the file with the client certificate in the PEM format.
-   -  **keyfile**: Specifies the file with the client secret key in the PEM format. If not provided patronictl will use the value provided for REST API "keyfile" parameter.
+   -  **keyfile**: Specifies the file with the client secret key in the PEM format.
-   -  **keyfile\_password**: Specifies a password for decrypting the keyfile. If not provided patronictl will use the value provided for REST API "keyfile\_password" parameter.
+   -  **keyfile\_password**: Specifies a password for decrypting the client keyfile.
 
 Watchdog
 --------

features/Dockerfile

@@ -27,8 +27,8 @@ RUN set -ex \
     && apt-get update \
     && apt-get reinstall init-system-helpers \
     && apt-get install -y \
-      python3-pip \
       python3-dev \
+      python3-venv \
       rsync \
       curl \
       gcc \
@@ -40,7 +40,9 @@ RUN set -ex \
       net-tools \
       iputils-ping \
     && rm -rf /var/cache/apt \
-    && python3 -m pip install --no-cache-dir tox \
+    \
+    && python3 -m venv /tox \
+    && /tox/bin/pip install --no-cache-dir tox>=4 \
     \
     && mkdir -p "$PGHOME" \
     && sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \
@@ -50,6 +52,7 @@ RUN set -ex \
     && curl -sL "$ETCDURL/etcd-v$ETCDVERSION-linux-$(dpkg --print-architecture).tar.gz" \
        | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl
 
+ENV PATH="/tox/bin:$PATH"
 
 # This Dockerfile syntax only works with docker buildx and the syntax
 # line at the top of this file.

features/basic_replication.feature

@@ -79,6 +79,7 @@ Feature: basic replication
     When I add the table buz to postgres2
     Then table buz is present on postgres0 after 20 seconds
 
+  @reject-duplicate-name
   Scenario: check graceful rejection when two nodes have the same name
     Given I start duplicate postgres0 on port 8011
     Then there is a "Can't start; there is already a node named 'postgres0' running" CRITICAL in the dup-postgres0 patroni log

features/environment.py

@@ -59,7 +59,7 @@ class AbstractController(abc.ABC):
                 break
             time.sleep(1)
         else:
-            assert False,\
+            assert False, \
                 "{0} instance is not available for queries after {1} seconds".format(self._name, max_wait_limit)
 
     def stop(self, kill=False, timeout=15, _=False):
@@ -1082,7 +1082,9 @@ def before_all(context):
                            'PATRONI_RESTAPI_CERTFILE': context.certfile,
                            'PATRONI_RESTAPI_KEYFILE': context.keyfile,
                            'PATRONI_RESTAPI_VERIFY_CLIENT': 'required',
-                           'PATRONI_CTL_INSECURE': 'on'})
+                           'PATRONI_CTL_INSECURE': 'on',
+                           'PATRONI_CTL_CERTFILE': context.certfile,
+                           'PATRONI_CTL_KEYFILE': context.keyfile})
         ctl.update({'cacert': context.certfile, 'certfile': context.certfile, 'keyfile': context.keyfile})
     context.request_executor = PatroniRequest({'ctl': ctl}, True)
     context.dcs_ctl = context.pctl.known_dcs[context.pctl.dcs](context)
@@ -1144,3 +1146,5 @@ def before_scenario(context, scenario):
                 break
     if 'dcs-failsafe' in scenario.effective_tags and not context.dcs_ctl._handle:
         scenario.skip('it is not possible to control state of {0} from tests'.format(context.dcs_ctl.name()))
+    if 'reject-duplicate-name' in scenario.effective_tags and context.dcs_ctl.name() == 'raft':
+        scenario.skip('Flaky test with Raft')

features/steps/basic_replication.py

@@ -21,7 +21,7 @@ def start_duplicate_patroni(context, name, port):
         context.pctl.start('dup-' + name, custom_config=config)
         assert False, "Process was expected to fail"
     except AssertionError as e:
-        assert 'is not running after being started' in str(e),\
+        assert 'is not running after being started' in str(e), \
             "No error was raised by duplicate start of {0} ".format(name)
 
 
@@ -88,14 +88,14 @@ def table_is_present_on(context, table_name, pg_name, max_replication_delay):
             break
         sleep(1)
     else:
-        assert False,\
+        assert False, \
             "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay)
 
 
 @then('{pg_name:w} role is the {pg_role:w} after {max_promotion_timeout:d} seconds')
 def check_role(context, pg_name, pg_role, max_promotion_timeout):
     max_promotion_timeout *= context.timeout_multiplier
-    assert context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)),\
+    assert context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)), \
         "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
 
 
@@ -111,5 +111,5 @@ def replication_works(context, primary, replica, time_limit):
 @then('there is a "{message}" {level:w} in the {node} patroni log')
 def check_patroni_log(context, message, level, node):
     messsages_of_level = context.pctl.read_patroni_log(node, level)
-    assert any(message in line for line in messsages_of_level),\
+    assert any(message in line for line in messsages_of_level), \
         "There was no {0} {1} in the {2} patroni log".format(message, level, node)

features/steps/citus.py

@@ -125,5 +125,5 @@ def check_transaction(context, name):
 
 @step("a transaction finishes in {timeout:d} seconds")
 def check_transaction_timeout(context, timeout):
-    assert (datetime.now(tzutc) - context.xact_start).seconds > timeout,\
+    assert (datetime.now(tzutc) - context.xact_start).seconds > timeout, \
         "a transaction finished earlier than in {0} seconds".format(timeout)

features/steps/patroni_api.py

@@ -98,7 +98,7 @@ def do_run(context, cmd):
 @then('I receive a response {component:w} {data}')
 def check_response(context, component, data):
     if component == 'code':
-        assert context.status_code == int(data),\
+        assert context.status_code == int(data), \
             "status code {0} != {1}, response: {2}".format(context.status_code, data, context.response)
     elif component == 'returncode':
         assert context.status_code == int(data), "return code {0} != {1}, {2}".format(context.status_code,
@@ -158,7 +158,7 @@ def check_http_response(context, url, value, timeout, negate=False):
             break
         time.sleep(1)
     else:
-        assert False,\
+        assert False, \
             "Value {0} is {1} present in response after {2} seconds".format(value, "not" if not negate else "", timeout)
 
 

patroni/__main__.py

@@ -1,3 +1,8 @@
+"""Patroni main entry point.
+
+Implement ``patroni`` main daemon and expose its entry point.
+"""
+
 import logging
 import os
 import signal
@@ -8,6 +13,7 @@ from argparse import Namespace
 from typing import Any, Dict, Optional, TYPE_CHECKING
 
 from patroni.daemon import AbstractPatroniDaemon, abstract_main, get_base_arg_parser
+from patroni.tags import Tags
 
 if TYPE_CHECKING:  # pragma: no cover
     from .config import Config
@@ -15,9 +21,33 @@ if TYPE_CHECKING:  # pragma: no cover
 logger = logging.getLogger(__name__)
 
 
-class Patroni(AbstractPatroniDaemon):
+class Patroni(AbstractPatroniDaemon, Tags):
+    """Implement ``patroni`` command daemon.
+
+    :ivar version: Patroni version.
+    :ivar dcs: DCS object.
+    :ivar watchdog: watchdog handler, if configured to use watchdog.
+    :ivar postgresql: managed Postgres instance.
+    :ivar api: REST API server instance of this node.
+    :ivar request: wrapper for performing HTTP requests.
+    :ivar ha: HA handler.
+    :ivar next_run: time when to run the next HA loop cycle.
+    :ivar scheduled_restart: when a restart has been scheduled to occur, if any. In that case, should contain two keys:
+        * ``schedule``: timestamp when restart should occur;
+        * ``postmaster_start_time``: timestamp when Postgres was last started.
+    """
 
     def __init__(self, config: 'Config') -> None:
+        """Create a :class:`Patroni` instance with the given *config*.
+
+        Get a connection to the DCS, configure watchdog (if required), set up Patroni interface with Postgres, configure
+        the HA loop and bring the REST API up.
+
+        .. note::
+            Expected to be instantiated and run through :func:`~patroni.daemon.abstract_main`.
+
+        :param config: Patroni configuration.
+        """
         from patroni.api import RestApiServer
         from patroni.dcs import get_dcs
         from patroni.ha import Ha
@@ -41,11 +71,22 @@ class Patroni(AbstractPatroniDaemon):
         self.api = RestApiServer(self, self.config['restapi'])
         self.ha = Ha(self)
 
-        self.tags = self.get_tags()
+        self._tags = self._get_tags()
         self.next_run = time.time()
         self.scheduled_restart: Dict[str, Any] = {}
 
     def load_dynamic_configuration(self) -> None:
+        """Load Patroni dynamic configuration.
+
+        Load dynamic configuration from the DCS, if `/config` key is available in the DCS, otherwise fall back to
+        ``bootstrap.dcs`` section from the configuration file.
+
+        If the DCS connection fails returning the exception :class:`~patroni.exceptions.DCSError` an attempt will be
+        remade every 5 seconds.
+
+        .. note::
+            This method is called only once, at the time when Patroni is started.
+        """
         from patroni.exceptions import DCSError
         while True:
             try:
@@ -74,29 +115,37 @@ class Patroni(AbstractPatroniDaemon):
         if not isinstance(member, Member):
             return
         try:
-            _ = self.request(member, endpoint="/liveness")
+            _ = self.request(member, endpoint="/liveness", timeout=3)
             logger.fatal("Can't start; there is already a node named '%s' running", self.config['name'])
             sys.exit(1)
         except Exception:
             return
 
-    def get_tags(self) -> Dict[str, Any]:
+    def _get_tags(self) -> Dict[str, Any]:
-        return {tag: value for tag, value in self.config.get('tags', {}).items()
+        """Get tags configured for this node, if any.
-                if tag not in ('clonefrom', 'nofailover', 'noloadbalance', 'nosync') or value}
 
-    @property
+        :returns: a dictionary of tags set for this node.
-    def nofailover(self) -> bool:
+        """
-        return bool(self.tags.get('nofailover', False))
+        return self._filter_tags(self.config.get('tags', {}))
-
-    @property
-    def nosync(self) -> bool:
-        return bool(self.tags.get('nosync', False))
 
     def reload_config(self, sighup: bool = False, local: Optional[bool] = False) -> None:
+        """Apply new configuration values for ``patroni`` daemon.
+
+        Reload:
+            * Cached tags;
+            * Request wrapper configuration;
+            * REST API configuration;
+            * Watchdog configuration;
+            * Postgres configuration;
+            * DCS configuration.
+
+        :param sighup: if it is related to a SIGHUP signal.
+        :param local: if there has been changes to the local configuration file.
+        """
         try:
             super(Patroni, self).reload_config(sighup, local)
             if local:
-                self.tags = self.get_tags()
+                self._tags = self._get_tags()
                 self.request.reload_config(self.config)
             if local or sighup and self.api.reload_local_certificate():
                 self.api.reload_config(self.config['restapi'])
@@ -107,14 +156,16 @@ class Patroni(AbstractPatroniDaemon):
             logger.exception('Failed to reload config_file=%s', self.config.config_file)
 
     @property
-    def replicatefrom(self):
+    def tags(self) -> Dict[str, Any]:
-        return self.tags.get('replicatefrom')
+        """Tags configured for this node, if any."""
-
+        return self._tags
-    @property
-    def noloadbalance(self):
-        return bool(self.tags.get('noloadbalance', False))
 
     def schedule_next_run(self) -> None:
+        """Schedule the next run of the ``patroni`` daemon main loop.
+
+        Next run is scheduled based on previous run plus value of ``loop_wait`` configuration from DCS. If that has
+        already been exceeded, run the next cycle immediately.
+        """
         self.next_run += self.dcs.loop_wait
         current_time = time.time()
         nap_time = self.next_run - current_time
@@ -128,11 +179,21 @@ class Patroni(AbstractPatroniDaemon):
             self.next_run = time.time()
 
     def run(self) -> None:
+        """Run ``patroni`` daemon process main loop.
+
+        Start the REST API and keep running HA cycles every ``loop_wait`` seconds.
+        """
         self.api.start()
         self.next_run = time.time()
         super(Patroni, self).run()
 
     def _run_cycle(self) -> None:
+        """Run a cycle of the ``patroni`` daemon main loop.
+
+        Run an HA cycle and schedule the next cycle run. If any dynamic configuration change request is detected, apply
+        the change and cache the new dynamic configuration values in ``patroni.dynamic.json`` file under Postgres data
+        directory.
+        """
         logger.info(self.ha.run_cycle())
 
         if self.dcs.cluster and self.dcs.cluster.config and self.dcs.cluster.config.data \
@@ -145,6 +206,10 @@ class Patroni(AbstractPatroniDaemon):
         self.schedule_next_run()
 
     def _shutdown(self) -> None:
+        """Perform shutdown of ``patroni`` daemon process.
+
+        Shut down the REST API and the HA handler.
+        """
         try:
             self.api.shutdown()
         except Exception:
@@ -156,18 +221,54 @@ class Patroni(AbstractPatroniDaemon):
 
 
 def patroni_main(configfile: str) -> None:
+    """Configure and start ``patroni`` main daemon process.
+
+    :param configfile: path to Patroni configuration file.
+    """
     from multiprocessing import freeze_support
 
+    # Windows executables created by PyInstaller are frozen, thus we need to enable frozen support for
+    # :mod:`multiprocessing` to avoid :class:`RuntimeError` exceptions.
     freeze_support()
     abstract_main(Patroni, configfile)
 
 
 def process_arguments() -> Namespace:
+    """Process command-line arguments.
+
+    Create a basic command-line parser through :func:`~patroni.daemon.get_base_arg_parser`, extend its capabilities by
+    adding these flags and parse command-line arguments.:
+
+      * ``--validate-config`` -- used to validate the Patroni configuration file
+      * ``--generate-config`` -- used to generate Patroni configuration from a running PostgreSQL instance
+      * ``--generate-sample-config`` -- used to generate a sample Patroni configuration
+
+    .. note::
+        If running with ``--generate-config``, ``--generate-sample-config`` or ``--validate-flag`` will exit
+        after generating or validating configuration.
+
+    :returns: parsed arguments, if not running with ``--validate-config`` flag.
+    """
+    from patroni.config_generator import generate_config
+
     parser = get_base_arg_parser()
-    parser.add_argument('--validate-config', action='store_true', help='Run config validator and exit')
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument('--validate-config', action='store_true', help='Run config validator and exit')
+    group.add_argument('--generate-sample-config', action='store_true',
+                       help='Generate a sample Patroni yaml configuration file')
+    group.add_argument('--generate-config', action='store_true',
+                       help='Generate a Patroni yaml configuration file for a running instance')
+    parser.add_argument('--dsn', help='Optional DSN string of the instance to be used as a source \
+                                    for config generation. Superuser connection is required.')
     args = parser.parse_args()
 
-    if args.validate_config:
+    if args.generate_sample_config:
+        generate_config(args.configfile, True, None)
+        sys.exit(0)
+    elif args.generate_config:
+        generate_config(args.configfile, False, args.dsn)
+        sys.exit(0)
+    elif args.validate_config:
         from patroni.validator import schema
         from patroni.config import Config, ConfigParseError
 
@@ -181,6 +282,16 @@ def process_arguments() -> Namespace:
 
 
 def main() -> None:
+    """Main entrypoint of :mod:`patroni.__main__`.
+
+    Process command-line arguments, ensure :mod:`psycopg2` (or :mod:`psycopg`) attendee the pre-requisites and start
+    ``patroni`` daemon process.
+
+    .. note::
+        If running through a Docker container, make the main process take care of init process duties and run
+        ``patroni`` daemon as another process. In that case relevant signals received by the main process and forwarded
+        to ``patroni`` daemon process.
+    """
     from patroni import check_psycopg
 
     args = process_arguments()
@@ -196,7 +307,13 @@ def main() -> None:
 
     # Looks like we are in a docker, so we will act like init
     def sigchld_handler(signo: int, stack_frame: Optional[FrameType]) -> None:
+        """Handle ``SIGCHLD`` received by main process from ``patroni`` daemon when the daemon terminates.
+
+        :param signo: signal number.
+        :param stack_frame: current stack frame.
+        """
         try:
+            # log exit code of all children processes, and break loop when there is none left
             while True:
                 ret = os.waitpid(-1, os.WNOHANG)
                 if ret == (0, 0):
@@ -206,7 +323,12 @@ def main() -> None:
         except OSError:
             pass
 
-    def passtochild(signo: int, stack_frame: Optional[FrameType]):
+    def passtochild(signo: int, stack_frame: Optional[FrameType]) -> None:
+        """Forward a signal *signo* from main process to child process.
+
+        :param signo: signal number.
+        :param stack_frame: current stack frame.
+        """
         if pid:
             os.kill(pid, signo)
 

patroni/api.py

@@ -12,7 +12,6 @@ import json
 import logging
 import time
 import traceback
-import dateutil.parser
 import datetime
 import os
 import socket
@@ -28,11 +27,11 @@ from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, TYPE_CH
 
 from . import psycopg
 from .__main__ import Patroni
-from .dcs import Cluster
 from .exceptions import PostgresConnectionException, PostgresException
+from .manual_failover import ManualFailover
 from .postgresql.misc import postgres_version_to_int
 from .utils import deep_compare, enable_keepalive, parse_bool, patch_config, Retry, \
-    RetryFailedError, parse_int, split_host_port, tzutc, uri, cluster_as_json
+    RetryFailedError, parse_int, parse_schedule, split_host_port, tzutc, uri, cluster_as_json
 
 logger = logging.getLogger(__name__)
 
@@ -49,9 +48,23 @@ def check_access(func: Callable[['RestApiHandler'], None]) -> Callable[..., None
 
     :Example:
 
-        @check_access
+        >>> class FooServer:
-        def do_PUT_foo():
+        ...   def check_access(self, *args, **kwargs):
-            pass
+        ...     print(f'In FooServer: {args[0].__class__.__name__}')
+        ...     return True
+        ...
+
+        >>> class Foo:
+        ...   server = FooServer()
+        ...   @check_access
+        ...   def do_PUT_foo(self):
+        ...      print('In do_PUT_foo')
+
+        >>> f = Foo()
+        >>> f.do_PUT_foo()
+        In FooServer: Foo
+        In do_PUT_foo
+
     """
 
     def wrapper(self: 'RestApiHandler', *args: Any, **kwargs: Any) -> None:
@@ -97,6 +110,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         """Write a response that is composed only of the HTTP status.
 
         The response is written with these values separated by space:
+
             * HTTP protocol version;
             * *status_code*;
             * description of *status_code*.
@@ -157,19 +171,19 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Modifies *response* before sending it to the client. Defines the ``patroni`` key, which is a
         dictionary that contains the mandatory keys:
 
-        * ``version``: Patroni version, e.g. ``3.0.2``;
+            * ``version``: Patroni version, e.g. ``3.0.2``;
-        * ``scope``: value of ``scope`` setting from Patroni configuration.
+            * ``scope``: value of ``scope`` setting from Patroni configuration.
 
         May also add the following optional keys, depending on the status of this Patroni/PostgreSQL node:
 
-        * ``tags``: tags that were set through Patroni configuration merged with dynamically applied tags;
+            * ``tags``: tags that were set through Patroni configuration merged with dynamically applied tags;
-        * ``database_system_identifier``: ``Database system identifier`` from ``pg_controldata`` output;
+            * ``database_system_identifier``: ``Database system identifier`` from ``pg_controldata`` output;
-        * ``pending_restart``: ``True`` if PostgreSQL is pending to be restarted;
+            * ``pending_restart``: ``True`` if PostgreSQL is pending to be restarted;
-        * ``scheduled_restart``: a dictionary with a single key ``schedule``, which is the timestamp for the scheduled
+            * ``scheduled_restart``: a dictionary with a single key ``schedule``, which is the timestamp for the
-            restart;
+                scheduled restart;
-        * ``watchdog_failed``: ``True`` if watchdog device is unhealthy;
+            * ``watchdog_failed``: ``True`` if watchdog device is unhealthy;
-        * ``logger_queue_size``: log queue length if it is longer than expected;
+            * ``logger_queue_size``: log queue length if it is longer than expected;
-        * ``logger_records_lost``: number of log records that have been lost while the log queue was full.
+            * ``logger_records_lost``: number of log records that have been lost while the log queue was full.
 
         :param status_code: response HTTP status code.
         :param response: represents the status of the PostgreSQL node, and is used as a basis for the HTTP response.
@@ -183,7 +197,11 @@ class RestApiHandler(BaseHTTPRequestHandler):
             response['database_system_identifier'] = patroni.postgresql.sysid
         if patroni.postgresql.pending_restart:
             response['pending_restart'] = True
-        response['patroni'] = {'version': patroni.version, 'scope': patroni.postgresql.scope}
+        response['patroni'] = {
+            'version': patroni.version,
+            'scope': patroni.postgresql.scope,
+            'name': patroni.postgresql.name
+        }
         if patroni.scheduled_restart:
             response['scheduled_restart'] = patroni.scheduled_restart.copy()
             del response['scheduled_restart']['postmaster_start_time']
@@ -204,32 +222,54 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Is used for handling all health-checks requests. E.g. "GET /(primary|replica|sync|async|etc...)".
 
         The (optional) query parameters and the HTTP response status depend on the requested path:
+
             * ``/``, ``primary``, or ``read-write``:
+
                 * HTTP status ``200``: if a primary with the leader lock.
+
             * ``/standby-leader``:
+
                 * HTTP status ``200``: if holds the leader lock in a standby cluster.
+
             * ``/leader``:
+
                 * HTTP status ``200``: if holds the leader lock.
+
             * ``/replica``:
+
                 * Query parameters:
+
                     * ``lag``: only accept replication lag up to ``lag``. Accepts either an :class:`int`, which
                         represents lag in bytes, or a :class:`str` representing lag in human-readable format (e.g.
                         ``10MB``).
                     * Any custom parameter: will attempt to match them against node tags.
+
                 * HTTP status ``200``: if up and running as a standby and without ``noloadbalance`` tag.
+
             * ``/read-only``:
+
                 * HTTP status ``200``: if up and running and without ``noloadbalance`` tag.
+
             * ``/synchronous`` or ``/sync``:
+
                 * HTTP status ``200``: if up and running as a synchronous standby.
+
             * ``/read-only-sync``:
+
                 * HTTP status ``200``: if up and running as a synchronous standby or primary.
+
             * ``/asynchronous``:
+
                 * Query parameters:
+
                     * ``lag``: only accept replication lag up to ``lag``. Accepts either an :class:`int`, which
                         represents lag in bytes, or a :class:`str` representing lag in human-readable format (e.g.
                         ``10MB``).
+
                 * HTTP status ``200``: if up and running as an asynchronous standby.
+
             * ``/health``:
+
                 * HTTP status ``200``: if up and running.
 
         .. note::
@@ -333,16 +373,16 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_OPTIONS(self) -> None:
         """Handle an ``OPTIONS`` request.
 
-        Write a simple HTTP response that represents the current PostgreSQL status. Send only `200 OK` or
+        Write a simple HTTP response that represents the current PostgreSQL status. Send only ``200 OK`` or
-        `503 Service Unavailable` as a response and nothing more, particularly no headers.
+        ``503 Service Unavailable`` as a response and nothing more, particularly no headers.
         """
         self.do_GET(write_status_code_only=True)
 
     def do_HEAD(self) -> None:
         """Handle a ``HEAD`` request.
 
-        Write a simple HTTP response that represents the current PostgreSQL status. Send only `200 OK` or
+        Write a simple HTTP response that represents the current PostgreSQL status. Send only ``200 OK`` or
-        `503 Service Unavailable` as a response and nothing more, particularly no headers.
+        ``503 Service Unavailable`` as a response and nothing more, particularly no headers.
         """
         self.do_GET(write_status_code_only=True)
 
@@ -350,11 +390,17 @@ class RestApiHandler(BaseHTTPRequestHandler):
         """Handle a ``GET`` request to ``/liveness`` path.
 
         Write a simple HTTP response with HTTP status:
+
             * ``200``:
+
                 * If the cluster is in maintenance mode; or
                 * If Patroni heartbeat loop is properly running;
-            * ``503`` if Patroni heartbeat loop last run was more than ``ttl`` setting ago on the primary (or twice the
+
-                value of ``ttl`` on a replica).
+            * ``503``:
+
+                * if Patroni heartbeat loop last run was more than ``ttl`` setting ago on the primary (or twice the
+                    value of ``ttl`` on a replica).
+
         """
         patroni: Patroni = self.server.patroni
         is_primary = patroni.postgresql.role in ('master', 'primary') and patroni.postgresql.is_running()
@@ -371,10 +417,14 @@ class RestApiHandler(BaseHTTPRequestHandler):
         """Handle a ``GET`` request to ``/readiness`` path.
 
         Write a simple HTTP response which HTTP status can be:
+
             * ``200``:
+
                 * If this Patroni node holds the DCS leader lock; or
                 * If this PostgreSQL instance is up and running;
+
             * ``503``: if none of the previous conditions apply.
+
         """
         patroni = self.server.patroni
         if patroni.ha.is_leader():
@@ -397,12 +447,15 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_GET_cluster(self) -> None:
         """Handle a ``GET`` request to ``/cluster`` path.
 
-        Write an HTTP response with JSON content based on the output of :func:`cluster_as_json`, with HTTP status
+        Write an HTTP response with JSON content based on the output of :func:`~patroni.utils.cluster_as_json`, with
-        ``200`` and the JSON representation of the cluster topology.
+        HTTP status ``200`` and the JSON representation of the cluster topology.
         """
         cluster = self.server.patroni.dcs.get_cluster(True)
         global_config = self.server.patroni.config.get_global_config(cluster)
-        self._write_json_response(200, cluster_as_json(cluster, global_config))
+
+        response = cluster_as_json(cluster, global_config)
+        response['scope'] = self.server.patroni.postgresql.scope
+        self._write_json_response(200, response)
 
     def do_GET_history(self) -> None:
         """Handle a ``GET`` request to ``/history`` path.
@@ -412,11 +465,13 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         The response contains a :class:`list` of failover/switchover events. Each item is a :class:`list` with the
         following items:
+
             * Timeline when the event occurred (class:`int`);
             * LSN at which the event occurred (class:`int`);
             * The reason for the event (class:`str`);
             * Timestamp when the new timeline was created (class:`str`);
             * Name of the involved Patroni node (class:`str`).
+
         """
         cluster = self.server.patroni.dcs.cluster or self.server.patroni.dcs.get_cluster()
         self._write_json_response(200, cluster.history and cluster.history.lines or [])
@@ -443,26 +498,33 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         The response contains the following items:
 
-        * ``patroni_version``: Patroni version without periods, e.g. ``030002`` for Patroni ``3.0.2``;
+            * ``patroni_version``: Patroni version without periods, e.g. ``030002`` for Patroni ``3.0.2``;
-        * ``patroni_postgres_running``: ``1`` if PostgreSQL is running, else ``0``;
+            * ``patroni_postgres_running``: ``1`` if PostgreSQL is running, else ``0``;
-        * ``patroni_postmaster_start_time``: epoch timestamp since Postmaster was started;
+            * ``patroni_postmaster_start_time``: epoch timestamp since Postmaster was started;
-        * ``patroni_master``: ``1`` if this node holds the leader lock, else ``0``;
+            * ``patroni_master``: ``1`` if this node holds the leader lock, else ``0``;
-        * ``patroni_primary``: same as ``patroni_master``;
+            * ``patroni_primary``: same as ``patroni_master``;
-        * ``patroni_xlog_location``: ``pg_wal_lsn_diff(pg_current_wal_lsn(), '0/0')`` if leader, else ``0``;
+            * ``patroni_xlog_location``: ``pg_wal_lsn_diff(pg_current_wal_flush_lsn(), '0/0')`` if leader, else ``0``;
-        * ``patroni_standby_leader``: ``1`` if standby leader node, else ``0``;
+            * ``patroni_standby_leader``: ``1`` if standby leader node, else ``0``;
-        * ``patroni_replica``: ``1`` if a replica, else ``0``;
+            * ``patroni_replica``: ``1`` if a replica, else ``0``;
-        * ``patroni_sync_standby``: ``1`` if a sync replica, else ``0``;
+            * ``patroni_sync_standby``: ``1`` if a sync replica, else ``0``;
-        * ``patroni_xlog_received_location``: ``pg_wal_lsn_diff(pg_last_wal_receive_lsn(), '0/0')``;
+            * ``patroni_xlog_received_location``: ``pg_wal_lsn_diff(pg_last_wal_receive_lsn(), '0/0')``;
-        * ``patroni_xlog_replayed_location``: ``pg_wal_lsn_diff(pg_last_wal_replay_lsn(), '0/0)``;
+            * ``patroni_xlog_replayed_location``: ``pg_wal_lsn_diff(pg_last_wal_replay_lsn(), '0/0)``;
-        * ``patroni_xlog_replayed_timestamp``: ``pg_last_xact_replay_timestamp``;
+            * ``patroni_xlog_replayed_timestamp``: ``pg_last_xact_replay_timestamp``;
-        * ``patroni_xlog_paused``: ``pg_is_wal_replay_paused()``;
+            * ``patroni_xlog_paused``: ``pg_is_wal_replay_paused()``;
-        * ``patroni_postgres_server_version``: Postgres version without periods, e.g. ``150002`` for Postgres ``15.2``;
+            * ``patroni_postgres_server_version``: Postgres version without periods, e.g. ``150002`` for Postgres
-        * ``patroni_cluster_unlocked``: ``1`` if no one holds the leader lock, else ``0``;
+              ``15.2``;
-        * ``patroni_failsafe_mode_is_active``: ``1`` if ``failsafe_mode`` is currently active, else ``0``;
+            * ``patroni_cluster_unlocked``: ``1`` if no one holds the leader lock, else ``0``;
-        * ``patroni_postgres_timeline``: PostgreSQL timeline based on current WAL file name;
+            * ``patroni_failsafe_mode_is_active``: ``1`` if ``failsafe_mode`` is currently active, else ``0``;
-        * ``patroni_dcs_last_seen``: epoch timestamp when DCS was last contacted successfully;
+            * ``patroni_postgres_timeline``: PostgreSQL timeline based on current WAL file name;
-        * ``patroni_pending_restart``: ``1`` if this PostgreSQL node is pending a restart, else ``0``;
+            * ``patroni_dcs_last_seen``: epoch timestamp when DCS was last contacted successfully;
-        * ``patroni_is_paused``: ``1`` if Patroni is in maintenance node, else ``0``.
+            * ``patroni_pending_restart``: ``1`` if this PostgreSQL node is pending a restart, else ``0``;
+            * ``patroni_is_paused``: ``1`` if Patroni is in maintenance node, else ``0``.
+
+        For PostgreSQL v9.6+ the response will also have the following:
+
+            * ``patroni_postgres_streaming``: 1 if Postgres is streaming from another node, else ``0``;
+            * ``patroni_postgres_in_archive_recovery``: ``1`` if Postgres isn't streaming and
+              there is ``restore_command`` available, else ``0``.
         """
         postgres = self.get_postgresql_status(True)
         patroni = self.server.patroni
@@ -470,113 +532,113 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         metrics: List[str] = []
 
-        scope_label = '{{scope="{0}"}}'.format(patroni.postgresql.scope)
+        labels = f'{{scope="{patroni.postgresql.scope}",name="{patroni.postgresql.name}"}}'
         metrics.append("# HELP patroni_version Patroni semver without periods.")
         metrics.append("# TYPE patroni_version gauge")
         padded_semver = ''.join([x.zfill(2) for x in patroni.version.split('.')])  # 2.0.2 => 020002
-        metrics.append("patroni_version{0} {1}".format(scope_label, padded_semver))
+        metrics.append("patroni_version{0} {1}".format(labels, padded_semver))
 
         metrics.append("# HELP patroni_postgres_running Value is 1 if Postgres is running, 0 otherwise.")
         metrics.append("# TYPE patroni_postgres_running gauge")
-        metrics.append("patroni_postgres_running{0} {1}".format(scope_label, int(postgres['state'] == 'running')))
+        metrics.append("patroni_postgres_running{0} {1}".format(labels, int(postgres['state'] == 'running')))
 
         metrics.append("# HELP patroni_postmaster_start_time Epoch seconds since Postgres started.")
         metrics.append("# TYPE patroni_postmaster_start_time gauge")
         postmaster_start_time = postgres.get('postmaster_start_time')
         postmaster_start_time = (postmaster_start_time - epoch).total_seconds() if postmaster_start_time else 0
-        metrics.append("patroni_postmaster_start_time{0} {1}".format(scope_label, postmaster_start_time))
+        metrics.append("patroni_postmaster_start_time{0} {1}".format(labels, postmaster_start_time))
 
         metrics.append("# HELP patroni_master Value is 1 if this node is the leader, 0 otherwise.")
         metrics.append("# TYPE patroni_master gauge")
-        metrics.append("patroni_master{0} {1}".format(scope_label, int(postgres['role'] in ('master', 'primary'))))
+        metrics.append("patroni_master{0} {1}".format(labels, int(postgres['role'] in ('master', 'primary'))))
 
         metrics.append("# HELP patroni_primary Value is 1 if this node is the leader, 0 otherwise.")
         metrics.append("# TYPE patroni_primary gauge")
-        metrics.append("patroni_primary{0} {1}".format(scope_label, int(postgres['role'] in ('master', 'primary'))))
+        metrics.append("patroni_primary{0} {1}".format(labels, int(postgres['role'] in ('master', 'primary'))))
 
         metrics.append("# HELP patroni_xlog_location Current location of the Postgres"
                        " transaction log, 0 if this node is not the leader.")
         metrics.append("# TYPE patroni_xlog_location counter")
-        metrics.append("patroni_xlog_location{0} {1}".format(scope_label, postgres.get('xlog', {}).get('location', 0)))
+        metrics.append("patroni_xlog_location{0} {1}".format(labels, postgres.get('xlog', {}).get('location', 0)))
 
         metrics.append("# HELP patroni_standby_leader Value is 1 if this node is the standby_leader, 0 otherwise.")
         metrics.append("# TYPE patroni_standby_leader gauge")
-        metrics.append("patroni_standby_leader{0} {1}".format(scope_label, int(postgres['role'] == 'standby_leader')))
+        metrics.append("patroni_standby_leader{0} {1}".format(labels, int(postgres['role'] == 'standby_leader')))
 
         metrics.append("# HELP patroni_replica Value is 1 if this node is a replica, 0 otherwise.")
         metrics.append("# TYPE patroni_replica gauge")
-        metrics.append("patroni_replica{0} {1}".format(scope_label, int(postgres['role'] == 'replica')))
+        metrics.append("patroni_replica{0} {1}".format(labels, int(postgres['role'] == 'replica')))
 
         metrics.append("# HELP patroni_sync_standby Value is 1 if this node is a sync standby replica, 0 otherwise.")
         metrics.append("# TYPE patroni_sync_standby gauge")
-        metrics.append("patroni_sync_standby{0} {1}".format(scope_label, int(postgres.get('sync_standby', False))))
+        metrics.append("patroni_sync_standby{0} {1}".format(labels, int(postgres.get('sync_standby', False))))
 
         metrics.append("# HELP patroni_xlog_received_location Current location of the received"
                        " Postgres transaction log, 0 if this node is not a replica.")
         metrics.append("# TYPE patroni_xlog_received_location counter")
         metrics.append("patroni_xlog_received_location{0} {1}"
-                       .format(scope_label, postgres.get('xlog', {}).get('received_location', 0)))
+                       .format(labels, postgres.get('xlog', {}).get('received_location', 0)))
 
         metrics.append("# HELP patroni_xlog_replayed_location Current location of the replayed"
                        " Postgres transaction log, 0 if this node is not a replica.")
         metrics.append("# TYPE patroni_xlog_replayed_location counter")
         metrics.append("patroni_xlog_replayed_location{0} {1}"
-                       .format(scope_label, postgres.get('xlog', {}).get('replayed_location', 0)))
+                       .format(labels, postgres.get('xlog', {}).get('replayed_location', 0)))
 
         metrics.append("# HELP patroni_xlog_replayed_timestamp Current timestamp of the replayed"
                        " Postgres transaction log, 0 if null.")
         metrics.append("# TYPE patroni_xlog_replayed_timestamp gauge")
         replayed_timestamp = postgres.get('xlog', {}).get('replayed_timestamp')
         replayed_timestamp = (replayed_timestamp - epoch).total_seconds() if replayed_timestamp else 0
-        metrics.append("patroni_xlog_replayed_timestamp{0} {1}".format(scope_label, replayed_timestamp))
+        metrics.append("patroni_xlog_replayed_timestamp{0} {1}".format(labels, replayed_timestamp))
 
         metrics.append("# HELP patroni_xlog_paused Value is 1 if the Postgres xlog is paused, 0 otherwise.")
         metrics.append("# TYPE patroni_xlog_paused gauge")
         metrics.append("patroni_xlog_paused{0} {1}"
-                       .format(scope_label, int(postgres.get('xlog', {}).get('paused', False) is True)))
+                       .format(labels, int(postgres.get('xlog', {}).get('paused', False) is True)))
 
         if postgres.get('server_version', 0) >= 90600:
             metrics.append("# HELP patroni_postgres_streaming Value is 1 if Postgres is streaming, 0 otherwise.")
             metrics.append("# TYPE patroni_postgres_streaming gauge")
             metrics.append("patroni_postgres_streaming{0} {1}"
-                           .format(scope_label, int(postgres.get('replication_state') == 'streaming')))
+                           .format(labels, int(postgres.get('replication_state') == 'streaming')))
 
             metrics.append("# HELP patroni_postgres_in_archive_recovery Value is 1"
                            " if Postgres is replicating from archive, 0 otherwise.")
             metrics.append("# TYPE patroni_postgres_in_archive_recovery gauge")
             metrics.append("patroni_postgres_in_archive_recovery{0} {1}"
-                           .format(scope_label, int(postgres.get('replication_state') == 'in archive recovery')))
+                           .format(labels, int(postgres.get('replication_state') == 'in archive recovery')))
 
         metrics.append("# HELP patroni_postgres_server_version Version of Postgres (if running), 0 otherwise.")
         metrics.append("# TYPE patroni_postgres_server_version gauge")
-        metrics.append("patroni_postgres_server_version {0} {1}".format(scope_label, postgres.get('server_version', 0)))
+        metrics.append("patroni_postgres_server_version {0} {1}".format(labels, postgres.get('server_version', 0)))
 
         metrics.append("# HELP patroni_cluster_unlocked Value is 1 if the cluster is unlocked, 0 if locked.")
         metrics.append("# TYPE patroni_cluster_unlocked gauge")
-        metrics.append("patroni_cluster_unlocked{0} {1}".format(scope_label, int(postgres.get('cluster_unlocked', 0))))
+        metrics.append("patroni_cluster_unlocked{0} {1}".format(labels, int(postgres.get('cluster_unlocked', 0))))
 
         metrics.append("# HELP patroni_failsafe_mode_is_active Value is 1 if failsafe mode is active, 0 if inactive.")
         metrics.append("# TYPE patroni_failsafe_mode_is_active gauge")
         metrics.append("patroni_failsafe_mode_is_active{0} {1}"
-                       .format(scope_label, int(postgres.get('failsafe_mode_is_active', 0))))
+                       .format(labels, int(postgres.get('failsafe_mode_is_active', 0))))
 
         metrics.append("# HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise.")
         metrics.append("# TYPE patroni_postgres_timeline counter")
-        metrics.append("patroni_postgres_timeline{0} {1}".format(scope_label, postgres.get('timeline', 0)))
+        metrics.append("patroni_postgres_timeline{0} {1}".format(labels, postgres.get('timeline', 0)))
 
         metrics.append("# HELP patroni_dcs_last_seen Epoch timestamp when DCS was last contacted successfully"
                        " by Patroni.")
         metrics.append("# TYPE patroni_dcs_last_seen gauge")
-        metrics.append("patroni_dcs_last_seen{0} {1}".format(scope_label, postgres.get('dcs_last_seen', 0)))
+        metrics.append("patroni_dcs_last_seen{0} {1}".format(labels, postgres.get('dcs_last_seen', 0)))
 
         metrics.append("# HELP patroni_pending_restart Value is 1 if the node needs a restart, 0 otherwise.")
         metrics.append("# TYPE patroni_pending_restart gauge")
         metrics.append("patroni_pending_restart{0} {1}"
-                       .format(scope_label, int(patroni.postgresql.pending_restart)))
+                       .format(labels, int(patroni.postgresql.pending_restart)))
 
         metrics.append("# HELP patroni_is_paused Value is 1 if auto failover is disabled, 0 otherwise.")
         metrics.append("# TYPE patroni_is_paused gauge")
-        metrics.append("patroni_is_paused{0} {1}".format(scope_label, int(postgres.get('pause', 0))))
+        metrics.append("patroni_is_paused{0} {1}".format(labels, int(postgres.get('pause', 0))))
 
         self.write_response(200, '\n'.join(metrics) + '\n', content_type='text/plain')
 
@@ -661,7 +723,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_POST_reload(self) -> None:
         """Handle a ``POST`` request to ``/reload`` path.
 
-        Schedules a reload to Patroni and writes a response with HTTP status `202`.
+        Schedules a reload to Patroni and writes a response with HTTP status ``202``.
         """
         self.server.patroni.sighup_handler()
         self.write_response(202, 'reload scheduled')
@@ -714,40 +776,6 @@ class RestApiHandler(BaseHTTPRequestHandler):
             self.server.patroni.api_sigterm()
         self.write_response(202, 'shutdown scheduled')
 
-    @staticmethod
-    def parse_schedule(schedule: str,
-                       action: str) -> Tuple[Union[int, None], Union[str, None], Union[datetime.datetime, None]]:
-        """Parse the given *schedule* and validate it.
-
-        :param schedule: a string representing a timestamp, e.g. ``2023-04-14T20:27:00+00:00``.
-        :param action: the action to be scheduled (``restart``, ``switchover``, or ``failover``).
-
-        :returns: a tuple composed of 3 items
-            * Suggested HTTP status code for a response:
-                * ``None``: if no issue was faced while parsing, leaving it up to the caller to decide the status; or
-                * ``400``: if no timezone information could be found in *schedule*; or
-                * ``422``: if *schedule* is invalid -- in the past or not parsable.
-            * An error message, if any error is faced, otherwise ``None``;
-            * Parsed *schedule*, if able to parse, otherwise ``None``.
-        """
-        error = None
-        scheduled_at = None
-        try:
-            scheduled_at = dateutil.parser.parse(schedule)
-            if scheduled_at.tzinfo is None:
-                error = 'Timezone information is mandatory for the scheduled {0}'.format(action)
-                status_code = 400
-            elif scheduled_at < datetime.datetime.now(tzutc):
-                error = 'Cannot schedule {0} in the past'.format(action)
-                status_code = 422
-            else:
-                status_code = None
-        except (ValueError, TypeError):
-            logger.exception('Invalid scheduled %s time: %s', action, schedule)
-            error = 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601'
-            status_code = 422
-        return status_code, error, scheduled_at
-
     @check_access
     def do_POST_restart(self) -> None:
         """Handle a ``POST`` request to ``/restart`` path.
@@ -755,25 +783,31 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Used to restart postgres (or schedule a restart), mainly by ``patronictl restart``.
 
         The request body should be a JSON dictionary, and it can contain the following keys:
+
             * ``schedule``: timestamp at which the restart should occur;
             * ``role``: restart only nodes which role is ``role``. Can be either:
+
                 * ``primary`` (or ``master``); or
                 * ``replica``.
+
             * ``postgres_version``: restart only nodes which PostgreSQL version is less than ``postgres_version``, e.g.
                 ``15.2``;
             * ``timeout``: if restart takes longer than ``timeout`` return an error and fail over to a replica;
             * ``restart_pending``: if we should restart only when have ``pending restart`` flag;
 
         Response HTTP status codes:
+
             * ``200``: if successfully performed an immediate restart; or
             * ``202``: if successfully scheduled a restart for later; or
             * ``500``: if the cluster is in maintenance mode; or
             * ``400``: if
+
                 * ``role`` value is invalid; or
                 * ``postgres_version`` value is invalid; or
                 * ``timeout`` is not a number, or lesser than ``0``; or
                 * request contains an unknown key; or
                 * exception is faced while performing an immediate restart.
+
             * ``409``: if another restart was already previously scheduled; or
             * ``503``: if any issue was found while performing an immediate restart; or
             * HTTP status returned by :func:`parse_schedule`, if any error was observed while parsing the schedule.
@@ -797,9 +831,9 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         for k in request:
             if k == 'schedule':
-                (_, data, request[k]) = self.parse_schedule(request[k], "restart")
+                parse_result, request[k] = parse_schedule(request[k])
-                if _:
+                if parse_result:
-                    status_code = _
+                    data, status_code = parse_result.value[0], parse_result.value[1]
                     break
             elif k == 'role':
                 if request[k] not in ('master', 'primary', 'replica'):
@@ -851,6 +885,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Used to remove a scheduled restart of PostgreSQL.
 
         Response HTTP status codes:
+
             * ``200``: if a scheduled restart was removed; or
             * ``404``: if no scheduled restart could be found.
         """
@@ -869,6 +904,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Used to remove a scheduled switchover in the cluster.
 
         It writes a response, and the HTTP status code can be:
+
             * ``200``: if a scheduled switchover was removed; or
             * ``404``: if no scheduled switchover could be found; or
             * ``409``: if not able to update the switchover info in the DCS.
@@ -890,11 +926,13 @@ class RestApiHandler(BaseHTTPRequestHandler):
         """Handle a ``POST`` request to ``/reinitialize`` path.
 
         The request body may contain a JSON dictionary with the following key:
+
             * ``force``: ``True`` if we want to cancel an already running task in order to reinit a replica.
 
         Response HTTP status codes:
+
             * ``200``: if the reinit operation has started; or
-            * ``503``: if any error is returned by :func:`Ha.reinitialize`.
+            * ``503``: if any error is returned by :func:`~patroni.ha.Ha.reinitialize`.
         """
         request = self._read_json_content(body_is_optional=True)
 
@@ -918,11 +956,15 @@ class RestApiHandler(BaseHTTPRequestHandler):
         :param candidate: name of the Patroni node to be promoted.
         :param action: the action that is ongoing (``switchover`` or ``failover``).
 
-        :returns: a tuple composed of 2 items
+        :returns: a tuple composed of 2 items:
+
             * Response HTTP status codes:
+
                 * ``200``: if the operation succeeded; or
                 * ``503``: if the operation failed or timed out.
+
             * A status message about the operation.
+
         """
         timeout = max(10, self.server.patroni.dcs.loop_wait)
         for _ in range(0, timeout * 2):
@@ -941,39 +983,6 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 logger.debug('Exception occurred during polling %s result: %s', action, e)
         return 503, action.title() + ' status unknown'
 
-    def is_failover_possible(self, cluster: Cluster, leader: Optional[str], candidate: Optional[str],
-                             action: str) -> Optional[str]:
-        """Checks whether there are nodes that could take over after demoting the primary.
-
-        :param cluster: the Patroni cluster.
-        :param leader: name of the current Patroni leader.
-        :param candidate: name of the Patroni node to be promoted.
-        :param action: the action to be performed (``switchover`` or ``failover``).
-
-        :returns: a string with the error message or ``None`` if good nodes are found.
-        """
-        is_synchronous_mode = self.server.patroni.config.get_global_config(cluster).is_synchronous_mode
-        if leader and (not cluster.leader or cluster.leader.name != leader):
-            return 'leader name does not match'
-        if candidate:
-            if action == 'switchover' and is_synchronous_mode and not cluster.sync.matches(candidate):
-                return 'candidate name does not match with sync_standby'
-            members = [m for m in cluster.members if m.name == candidate]
-            if not members:
-                return 'candidate does not exists'
-        elif is_synchronous_mode:
-            members = [m for m in cluster.members if cluster.sync.matches(m.name)]
-            if not members:
-                return action + ' is not possible: can not find sync_standby'
-        else:
-            members = [m for m in cluster.members if not cluster.leader or m.name != cluster.leader.name and m.api_url]
-            if not members:
-                return action + ' is not possible: cluster does not have members except leader'
-        for st in self.server.patroni.ha.fetch_nodes_statuses(members):
-            if st.failover_limitation() is None:
-                return None
-        return action + ' is not possible: no good candidates have been found'
-
     @check_access
     def do_POST_failover(self, action: str = 'failover') -> None:
         """Handle a ``POST`` request to ``/failover`` path.
@@ -981,17 +990,20 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Handles manual failovers/switchovers, mainly from ``patronictl``.
 
         The request body should be a JSON dictionary, and it can contain the following keys:
+
             * ``leader``: name of the current leader in the cluster;
             * ``candidate``: name of the Patroni node to be promoted;
             * ``scheduled_at``: a string representing the timestamp when to execute the switchover/failover, e.g.
                 ``2023-04-14T20:27:00+00:00``.
 
         Response HTTP status codes:
+
             * ``202``: if operation has been scheduled;
             * ``412``: if operation is not possible;
             * ``503``: if unable to register the operation to the DCS;
             * HTTP status returned by :func:`parse_schedule`, if any error was observed while parsing the schedule;
-            * HTTP status returned by :func:`poll_failover_result` if the operation has been processed immediately.
+            * HTTP status returned by :func:`poll_failover_result` if the operation has been processed immediately;
+            * ``400``: if none of the above applies.
 
         .. note::
             If unable to parse the request body, then the request is silently discarded.
@@ -999,7 +1011,6 @@ class RestApiHandler(BaseHTTPRequestHandler):
         :param action: the action to be performed (``switchover`` or ``failover``).
         """
         request = self._read_json_content()
-        (status_code, data) = (400, '')
         if not request:
             return
 
@@ -1012,26 +1023,15 @@ class RestApiHandler(BaseHTTPRequestHandler):
         logger.info("received %s request with leader=%s candidate=%s scheduled_at=%s",
                     action, leader, candidate, scheduled_at)
 
-        if action == 'failover' and not candidate:
+        manual_failover = ManualFailover(action, cluster, leader, candidate, scheduled_at,
-            data = 'Failover could be performed only to a specific candidate'
+                                         global_config.is_paused, global_config.is_synchronous_mode,
-        elif action == 'switchover' and not leader:
+                                         self.server.patroni)
-            data = 'Switchover could be performed only from a specific leader'
+        data, status_code = manual_failover.run_precheck().value
 
         if not data and scheduled_at:
-            if not leader:
+            parse_result, scheduled_at = manual_failover.parse_scheduled()
-                data = 'Scheduled {0} is possible only from a specific leader'.format(action)
+            if parse_result:
-            if not data and global_config.is_paused:
+                data, status_code = parse_result.value[0], parse_result.value[1]
-                data = "Can't schedule {0} in the paused state".format(action)
-            if not data:
-                (status_code, data, scheduled_at) = self.parse_schedule(scheduled_at, action)
-
-        if not data and global_config.is_paused and not candidate:
-            data = action.title() + ' is possible only to a specific candidate in a paused state'
-
-        if not data and not scheduled_at:
-            data = self.is_failover_possible(cluster, leader, candidate, action)
-            if data:
-                status_code = 412
 
         if not data:
             if self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
@@ -1043,14 +1043,12 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name,
                                                                   candidate, action)
             else:
-                data = 'failed to write {0} key into DCS'.format(action)
+                data = 'failed to write failover key into DCS'
                 status_code = 503
-        # pyright thinks ``status_code`` can be ``None`` because ``parse_schedule`` call may return ``None``. However,
+
-        # if that's the case, ``status_code`` will be overwritten somewhere between ``parse_schedule`` and
+        status_code = status_code or 400
-        # ``write_response`` calls.
+        self.write_response(status_code, data.format(action=action, leader=leader, candidate=candidate,
-        if TYPE_CHECKING:  # pragma: no cover
+                                                     cluster_name=self.server.patroni.postgresql.scope))
-            assert isinstance(status_code, int)
-        self.write_response(status_code, data)
 
     def do_POST_switchover(self) -> None:
         """Handle a ``POST`` request to ``/switchover`` path.
@@ -1063,8 +1061,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_POST_citus(self) -> None:
         """Handle a ``POST`` request to ``/citus`` path.
 
-        Call :func:`CitusHandler.handle_event` to handle the request, then write a response with HTTP status code
+        Call :func:`~patroni.postgresql.CitusHandler.handle_event` to handle the request, then write a response with
-        ``200``.
+        HTTP status code ``200``.
 
         .. note::
             If unable to parse the request body, then the request is silently discarded.
@@ -1080,18 +1078,21 @@ class RestApiHandler(BaseHTTPRequestHandler):
         self.write_response(200, 'OK')
 
     def parse_request(self) -> bool:
-        """Override :func:`parse_request` method to enrich basic functionality of :class:`BaseHTTPRequestHandler`.
+        """Override :func:`parse_request` to enrich basic functionality of :class:`~http.server.BaseHTTPRequestHandler`.
 
         Original class can only invoke :func:`do_GET`, :func:`do_POST`, :func:`do_PUT`, etc method implementations if
         they are defined.
+
         But we would like to have at least some simple routing mechanism, i.e.:
+
             * ``GET /uri1/part2`` request should invoke :func:`do_GET_uri1()`
             * ``POST /other`` should invoke :func:`do_POST_other()`
 
         If the :func:`do_<REQUEST_METHOD>_<first_part_url>` method does not exist we'll fall back to original behavior.
 
         :returns: ``True`` for success, ``False`` for failure; on failure, any relevant error response has already been
-        sent back.
+                  sent back.
+
         """
         ret = BaseHTTPRequestHandler.parse_request(self)
         if ret:
@@ -1104,20 +1105,18 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 self.command = mname
         return ret
 
-    def query(self, sql: str, *params: Any, **kwargs: Any) -> List[Tuple[Any, ...]]:
+    def query(self, sql: str, *params: Any, retry: bool = False) -> List[Tuple[Any, ...]]:
-        """Execute *sql* query with *params*.
+        """Execute *sql* query with *params* and optionally return results.
 
         :param sql: the SQL statement to be run.
         :param params: positional arguments to call :func:`RestApiServer.query` with.
-        :param kwargs: can contain the key ``retry``. If the key is present its value should be a :class:`bool` which
+        :param retry: whether the query should be retried upon failure or given up immediately.
-            indicates whether the query should be retried upon failure or given up immediately.
 
         :returns: a list of rows that were fetched from the database.
         """
-        if not kwargs.get('retry', False):
+        if not retry:
             return self.server.query(sql, *params)
-        retry = Retry(delay=1, retry_exceptions=PostgresConnectionException)
+        return Retry(delay=1, retry_exceptions=PostgresConnectionException)(self.server.query, sql, *params)
-        return retry(self.server.query, sql, *params)
 
     def get_postgresql_status(self, retry: bool = False) -> Dict[str, Any]:
         """Builds an object representing a status of "postgres".
@@ -1125,36 +1124,46 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Some of the values are collected by executing a query and other are taken from the state stored in memory.
 
         :param retry: whether the query should be retried if failed or give up immediately
+
         :returns: a dict with the status of Postgres/Patroni. The keys are:
+
             * ``state``: Postgres state among ``stopping``, ``stopped``, ``stop failed``, ``crashed``, ``running``,
-                ``starting``, ``start failed``, ``restarting``, ``restart failed``, ``initializing new cluster``,
+              ``starting``, ``start failed``, ``restarting``, ``restart failed``, ``initializing new cluster``,
-                ``initdb failed``, ``running custom bootstrap script``, ``custom bootstrap failed``,
+              ``initdb failed``, ``running custom bootstrap script``, ``custom bootstrap failed``,
-                ``creating replica``, or ``unknown``;
+              ``creating replica``, or ``unknown``;
             * ``postmaster_start_time``: ``pg_postmaster_start_time()``;
             * ``role``: ``replica`` or ``master`` based on ``pg_is_in_recovery()`` output;
             * ``server_version``: Postgres version without periods, e.g. ``150002`` for Postgres ``15.2``;
             * ``xlog``: dictionary. Its structure depends on ``role``:
+
                 * If ``master``:
-                    * ``location``: ``pg_current_wal_lsn()``
+
+                    * ``location``: ``pg_current_wal_flush_lsn()``
+
                 * If ``replica``:
+
                     * ``received_location``: ``pg_wal_lsn_diff(pg_last_wal_receive_lsn(), '0/0')``;
                     * ``replayed_location``: ``pg_wal_lsn_diff(pg_last_wal_replay_lsn(), '0/0)``;
                     * ``replayed_timestamp``: ``pg_last_xact_replay_timestamp``;
                     * ``paused``: ``pg_is_wal_replay_paused()``;
+
             * ``sync_standby``: ``True`` if replication mode is synchronous and this is a sync standby;
             * ``timeline``: PostgreSQL primary node timeline;
             * ``replication``: :class:`list` of :class:`dict` entries, one for each replication connection. Each entry
                 contains the following keys:
+
                 * ``application_name``: ``pg_stat_activity.application_name``;
                 * ``client_addr``: ``pg_stat_activity.client_addr``;
                 * ``state``: ``pg_stat_replication.state``;
                 * ``sync_priority``: ``pg_stat_replication.sync_priority``;
                 * ``sync_state``: ``pg_stat_replication.sync_state``;
                 * ``usename``: ``pg_stat_activity.usename``.
+
             * ``pause``: ``True`` if cluster is in maintenance mode;
             * ``cluster_unlocked``: ``True`` if cluster has no node holding the leader lock;
             * ``failsafe_mode_is_active``: ``True`` if DCS failsafe mode is currently active;
             * ``dcs_last_seen``: epoch timestamp DCS was last reached by Patroni.
+
         """
         postgresql = self.server.patroni.postgresql
         cluster = self.server.patroni.dcs.cluster
@@ -1173,8 +1182,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     " application_name, client_addr, w.state, sync_state, sync_priority"
                     " FROM pg_catalog.pg_stat_get_wal_senders() w, pg_catalog.pg_stat_get_activity(pid)) AS ri")
 
-            row = self.query(stmt.format(postgresql.wal_name, postgresql.lsn_name), retry=retry)[0]
+            row = self.query(stmt.format(postgresql.wal_name, postgresql.lsn_name,
-
+                                         postgresql.wal_flush), retry=retry)[0]
             result = {
                 'state': postgresql.state,
                 'postmaster_start_time': row[0],
@@ -1279,24 +1288,35 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         self.daemon = True
 
     def query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]:
-        """Execute *sql* query with *params*.
+        """Execute *sql* query with *params* and optionally return results.
+
+        .. note::
+            Prefer to use own connection to postgres and fallback to ``heartbeat`` when own isn't available.
 
         :param sql: the SQL statement to be run.
         :param params: positional arguments to be used as parameters for *sql*.
 
         :returns: a list of rows that were fetched from the database.
-        :raises psycopg.Error: if had issues while executing *sql*.
+
-        :raises PostgresConnectionException: if had issues while connecting to the database.
+        :raises:
+            :class:`psycopg.Error`: if had issues while executing *sql*.
+            :class:`~patroni.exceptions.PostgresConnectionException`: if had issues while connecting to the database.
         """
-        cursor = None
+        # We first try to get a heartbeat connection because it is always required for the main thread.
         try:
-            with self.patroni.postgresql.connection().cursor() as cursor:
+            heartbeat_connection = self.patroni.postgresql.connection_pool.get('heartbeat')
-                cursor.execute(sql.encode('utf-8'), params)
+            heartbeat_connection.get()  # try to open psycopg connection to postgres
-                return [r for r in cursor]
+        except psycopg.Error as exc:
-        except psycopg.Error as e:
+            raise PostgresConnectionException('connection problems') from exc
-            if cursor and cursor.connection.closed == 0:
+
-                raise e
+        try:
-            raise PostgresConnectionException('connection problems')
+            connection = self.patroni.postgresql.connection_pool.get('restapi')
+            connection.get()  # try to open psycopg connection to postgres
+        except psycopg.Error:
+            logger.debug('restapi connection to postgres is not available')
+            connection = heartbeat_connection
+
+        return connection.query(sql, *params)
 
     @staticmethod
     def _set_fd_cloexec(fd: socket.socket) -> None:
@@ -1346,7 +1366,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         :param host: hostname to be checked.
         :param port: port to be checked.
 
-        :rtype: Iterator[Union[IPv4Network, IPv6Network]] of *host* + *port* resolved to IP networks.
+        :yields: *host* + *port* resolved to IP networks.
         """
         try:
             for _, _, _, _, sa in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP):
@@ -1360,8 +1380,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         .. note::
             Only yields object if ``restapi.allowlist_include_members`` setting is enabled.
 
-        :rtype: Iterator[Union[IPv4Network, IPv6Network]] of each node ``restapi.connect_address`` resolved to an IP
+        :yields: each node ``restapi.connect_address`` resolved to an IP network.
-            network.
         """
         cluster = self.patroni.dcs.cluster
         if self.__allowlist_include_members and cluster:
@@ -1381,8 +1400,10 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         """Ensure client has enough privileges to perform a given request.
 
         Write a response back to the client if any issue is observed, and the HTTP status may be:
+
             * ``401``: if ``Authorization`` header is missing or contain an invalid password;
             * ``403``: if:
+
                 * ``restapi.allowlist`` was configured, but client IP is not in the allowed list; or
                 * ``restapi.allowlist_include_members`` is enabled, but client IP is not in the members list; or
                 * a client certificate is expected by the server, but is missing in the request.
@@ -1461,18 +1482,21 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         :param listen: IP and port to bind REST API to. It should be a string in the format ``host:port``, where
             ``host`` can be a hostname or IP address. It is the value of ``restapi.listen`` setting.
         :param ssl_options: dictionary that may contain the following keys, depending on what has been configured in
-            ``restapi` section:
+            ``restapi`` section:
+
             * ``certfile``: path to PEM certificate. If given, will start in HTTPS mode;
             * ``keyfile``: path to key of ``certfile``;
             * ``keyfile_password``: password for decrypting ``keyfile``;
             * ``cafile``: path to CA file to validate client certificates;
             * ``ciphers``: permitted cipher suites;
             * ``verify_client``: value can be one among:
+
                 * ``none``: do not check client certificates;
                 * ``optional``: check client certificate only for unsafe REST API endpoints;
                 * ``required``: check client certificate for all REST API endpoints.
 
-        :raises ValueError: if any issue is faced while parsing *listen*.
+        :raises:
+            :class:`ValueError`: if any issue is faced while parsing *listen*.
         """
         try:
             host, port = split_host_port(listen, None)
@@ -1520,7 +1544,8 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
                                client_address: Tuple[str, int]) -> None:
         """Process a request to the REST API.
 
-        Wrapper for :func:`ThreadingMixIn.process_request_thread` that additionally:
+        Wrapper for :func:`~socketserver.ThreadingMixIn.process_request_thread` that additionally:
+
             * Enable TCP keepalive
             * Perform SSL handshake (if an SSL socket).
 
@@ -1538,7 +1563,8 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
     def shutdown_request(self, request: Union[socket.socket, Tuple[bytes, socket.socket]]) -> None:
         """Shut down a request to the REST API.
 
-        Wrapper for :func:`HTTPServer.shutdown_request` that additionally:
+        Wrapper for :func:`http.server.HTTPServer.shutdown_request` that additionally:
+
             * Perform SSL shutdown handshake (if a SSL socket).
 
         :param request: socket to handle the client request.
@@ -1586,7 +1612,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         :param value: list of IPs and/or networks contained in ``restapi.allowlist`` setting. Each item can be a host,
             an IP, or a network in CIDR format.
 
-        :rtype: Iterator[Union[IPv4Network, IPv6Network]] of *host* + *port* resolved to IP networks.
+        :yields: *host* + *port* resolved to IP networks.
         """
         if isinstance(value, list):
             for v in value:
@@ -1603,7 +1629,9 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         """Reload REST API configuration.
 
         :param config: dictionary representing values under the ``restapi`` configuration section.
-        :raises ValueError: if ``listen`` key is not present in *config*.
+
+        :raises:
+            :class:`ValueError`: if ``listen`` key is not present in *config*.
         """
         if 'listen' not in config:  # changing config in runtime
             raise ValueError('Can not find "restapi.listen" config')

patroni/collections.py

@@ -3,7 +3,7 @@
 Provides a case insensitive :class:`dict` and :class:`set` object types.
 """
 from collections import OrderedDict
-from typing import Any, Collection, Dict, Iterator, MutableMapping, MutableSet, Optional
+from typing import Any, Collection, Dict, Iterator, KeysView, MutableMapping, MutableSet, Optional
 
 
 class CaseInsensitiveSet(MutableSet[str]):
@@ -187,6 +187,13 @@ class CaseInsensitiveDict(MutableMapping[str, Any]):
         """
         return CaseInsensitiveDict({v[0]: v[1] for v in self._values.values()})
 
+    def keys(self) -> KeysView[str]:
+        """Return a new view of the dict's keys.
+
+        :returns: a set-like object providing a view on the dict's keys
+        """
+        return self._values.keys()
+
     def __repr__(self) -> str:
         """Get a string representation of the dict.
 

patroni/config.py

@@ -1,3 +1,4 @@
+"""Facilities related to Patroni configuration."""
 import json
 import logging
 import os
@@ -13,6 +14,7 @@ from . import PATRONI_ENV_PREFIX
 from .collections import CaseInsensitiveDict
 from .dcs import ClusterConfig, Cluster
 from .exceptions import ConfigParseError
+from .file_perm import pg_perm
 from .postgresql.config import ConfigHandler
 from .utils import deep_compare, parse_bool, parse_int, patch_config
 
@@ -34,121 +36,162 @@ _AUTH_ALLOWED_PARAMETERS = (
 
 
 def default_validator(conf: Dict[str, Any]) -> List[str]:
+    """Ensure *conf* is not empty.
+
+    Designed to be used as default validator for :class:`Config` objects, if no specific validator is provided.
+
+    :param conf: configuration to be validated.
+
+    :returns: an empty list -- :class:`Config` expects the validator to return a list of 0 or more issues found while
+        validating the configuration.
+
+    :raises:
+        :class:`ConfigParseError`: if *conf* is empty.
+    """
     if not conf:
         raise ConfigParseError("Config is empty.")
     return []
 
 
 class GlobalConfig(object):
+    """A class that wraps global configuration and provides convenient methods to access/check values.
 
-    """A class that wrapps global configuration and provides convinient methods to access/check values.
+    It is instantiated either by calling :func:`get_global_config` or :meth:`Config.get_global_config`, which picks
-
+    either a configuration from provided :class:`Cluster` object (the most up-to-date) or from the
-    It is instantiated by calling :func:`Config.global_config` method which picks either a
+    local cache if :class:`ClusterConfig` is not initialized or doesn't have a valid config.
-    configuration from provided :class:`Cluster` object (the most up-to-date) or from the
-    local cache if :class::`ClusterConfig` is not initialized or doesn't have a valid config.
     """
 
     def __init__(self, config: Dict[str, Any]) -> None:
-        """Initialize :class:`GlobalConfig` object.
+        """Initialize :class:`GlobalConfig` object with given *config*.
 
         :param config: current configuration either from
-                       :class:`ClusterConfig` or from :class:`Config.dynamic_configuration`
+                       :class:`ClusterConfig` or from :func:`Config.dynamic_configuration`.
         """
         self.__config = config
 
     def get(self, name: str) -> Any:
-        """Gets global configuration value by name.
+        """Gets global configuration value by *name*.
 
-        :param name: parameter name
+        :param name: parameter name.
-        :returns: configuration value or `None` if it is missing
+
+        :returns: configuration value or ``None`` if it is missing.
         """
         return self.__config.get(name)
 
     def check_mode(self, mode: str) -> bool:
         """Checks whether the certain parameter is enabled.
 
-        :param mode: parameter name could be: synchronous_mode, failsafe_mode, pause, check_timeline, and so on
+        :param mode: parameter name, e.g. ``synchronous_mode``, ``failsafe_mode``, ``pause``, ``check_timeline``, and
-        :returns: `True` if *mode* is enabled in the global configuration.
+            so on.
+
+        :returns: ``True`` if parameter *mode* is enabled in the global configuration.
         """
         return bool(parse_bool(self.__config.get(mode)))
 
     @property
     def is_paused(self) -> bool:
-        """:returns: `True` if cluster is in maintenance mode."""
+        """``True`` if cluster is in maintenance mode."""
         return self.check_mode('pause')
 
     @property
     def is_synchronous_mode(self) -> bool:
-        """:returns: `True` if synchronous replication is requested."""
+        """``True`` if synchronous replication is requested."""
         return self.check_mode('synchronous_mode')
 
     @property
     def is_synchronous_mode_strict(self) -> bool:
-        """:returns: `True` if at least one synchronous node is required."""
+        """``True`` if at least one synchronous node is required."""
         return self.check_mode('synchronous_mode_strict')
 
     def get_standby_cluster_config(self) -> Union[Dict[str, Any], Any]:
-        """:returns: "standby_cluster" configuration."""
+        """Get ``standby_cluster`` configuration.
+
+        :returns: a copy of ``standby_cluster`` configuration.
+        """
         return deepcopy(self.get('standby_cluster'))
 
     @property
     def is_standby_cluster(self) -> bool:
-        """:returns: `True` if global configuration has a valid "standby_cluster" section."""
+        """``True`` if global configuration has a valid ``standby_cluster`` section."""
         config = self.get_standby_cluster_config()
         return isinstance(config, dict) and\
             bool(config.get('host') or config.get('port') or config.get('restore_command'))
 
     def get_int(self, name: str, default: int = 0) -> int:
-        """Gets current value from the global configuration and trying to return it as int.
+        """Gets current value of *name* from the global configuration and try to return it as :class:`int`.
 
-        :param name: name of the parameter
+        :param name: name of the parameter.
-        :param default: default value if *name* is not in the configuration or invalid
+        :param default: default value if *name* is not in the configuration or invalid.
-        :returns: currently configured value from the global configuration or *default* if it is not set or invalid.
+
+        :returns: currently configured value of *name* from the global configuration or *default* if it is not set or
+            invalid.
         """
         ret = parse_int(self.get(name))
         return default if ret is None else ret
 
     @property
     def min_synchronous_nodes(self) -> int:
-        """:returns: the minimal number of synchronous nodes based on whether strict mode is requested or not."""
+        """The minimal number of synchronous nodes based on whether ``synchronous_mode_strict`` is enabled or not."""
         return 1 if self.is_synchronous_mode_strict else 0
 
     @property
     def synchronous_node_count(self) -> int:
-        """:returns: currently configured value from the global configuration or 1 if it is not set or invalid."""
+        """Currently configured value of ``synchronous_node_count`` from the global configuration.
+
+        Assume ``1`` if it is not set or invalid.
+        """
         return max(self.get_int('synchronous_node_count', 1), self.min_synchronous_nodes)
 
     @property
     def maximum_lag_on_failover(self) -> int:
-        """:returns: currently configured value from the global configuration or 1048576 if it is not set or invalid."""
+        """Currently configured value of ``maximum_lag_on_failover`` from the global configuration.
+
+        Assume ``1048576`` if it is not set or invalid.
+        """
         return self.get_int('maximum_lag_on_failover', 1048576)
 
     @property
     def maximum_lag_on_syncnode(self) -> int:
-        """:returns: currently configured value from the global configuration or -1 if it is not set or invalid."""
+        """Currently configured value of ``maximum_lag_on_syncnode`` from the global configuration.
+
+        Assume ``-1`` if it is not set or invalid.
+        """
         return self.get_int('maximum_lag_on_syncnode', -1)
 
     @property
     def primary_start_timeout(self) -> int:
-        """:returns: currently configured value from the global configuration or 300 if it is not set or invalid."""
+        """Currently configured value of ``primary_start_timeout`` from the global configuration.
+
+        Assume ``300`` if it is not set or invalid.
+
+        .. note::
+            ``master_start_timeout`` is still supported to keep backward compatibility.
+        """
         default = 300
         return self.get_int('primary_start_timeout', default)\
             if 'primary_start_timeout' in self.__config else self.get_int('master_start_timeout', default)
 
     @property
     def primary_stop_timeout(self) -> int:
-        """:returns: currently configured value from the global configuration or 300 if it is not set or invalid."""
+        """Currently configured value of ``primary_stop_timeout`` from the global configuration.
+
+        Assume ``0`` if it is not set or invalid.
+
+        .. note::
+            ``master_stop_timeout`` is still supported to keep backward compatibility.
+        """
         default = 0
         return self.get_int('primary_stop_timeout', default)\
             if 'primary_stop_timeout' in self.__config else self.get_int('master_stop_timeout', default)
 
 
-def get_global_config(cluster: Union[Cluster, None], default: Optional[Dict[str, Any]] = None) -> GlobalConfig:
+def get_global_config(cluster: Optional[Cluster], default: Optional[Dict[str, Any]] = None) -> GlobalConfig:
     """Instantiates :class:`GlobalConfig` based on the input.
 
-    :param cluster: the currently known cluster state from DCS
+    :param cluster: the currently known cluster state from DCS.
-    :param default: default configuration, which will be used if there is no valid *cluster.config*
+    :param default: default configuration, which will be used if there is no valid *cluster.config*.
-    :returns: :class:`GlobalConfig` object
+
+    :returns: :class:`GlobalConfig` object.
     """
     # Try to protect from the case when DCS was wiped out
     if cluster and cluster.config and cluster.config.modify_version:
@@ -159,23 +202,29 @@ def get_global_config(cluster: Union[Cluster, None], default: Optional[Dict[str,
 
 
 class Config(object):
-    """
+    """Handle Patroni configuration.
+
     This class is responsible for:
 
-      1) Building and giving access to `effective_configuration` from:
+      1) Building and giving access to ``effective_configuration`` from:
-         * `Config.__DEFAULT_CONFIG` -- some sane default values
-         * `dynamic_configuration` -- configuration stored in DCS
-         * `local_configuration` -- configuration from `config.yml` or environment
 
-      2) Saving and loading `dynamic_configuration` into 'patroni.dynamic.json' file
+         * ``Config.__DEFAULT_CONFIG`` -- some sane default values;
+         * ``dynamic_configuration`` -- configuration stored in DCS;
+         * ``local_configuration`` -- configuration from `config.yml` or environment.
+
+      2) Saving and loading ``dynamic_configuration`` into 'patroni.dynamic.json' file
          located in local_configuration['postgresql']['data_dir'] directory.
-         This is necessary to be able to restore `dynamic_configuration`
+         This is necessary to be able to restore ``dynamic_configuration``
-         if DCS was accidentally wiped
+         if DCS was accidentally wiped.
 
-      3) Loading of configuration file in the old format and converting it into new format
+      3) Loading of configuration file in the old format and converting it into new format.
 
-      4) Mimicking some of the `dict` interfaces to make it possible
+      4) Mimicking some ``dict`` interfaces to make it possible
-         to work with it as with the old `config` object.
+         to work with it as with the old ``config`` object.
+
+    :cvar PATRONI_CONFIG_VARIABLE: name of the environment variable that can be used to load Patroni configuration from.
+    :cvar __CACHE_FILENAME: name of the file used to cache dynamic configuration under Postgres data directory.
+    :cvar __DEFAULT_CONFIG: default configuration values for some Patroni settings.
     """
 
     PATRONI_CONFIG_VARIABLE = PATRONI_ENV_PREFIX + 'CONFIGURATION'
@@ -193,21 +242,38 @@ class Config(object):
             'recovery_min_apply_delay': ''
         },
         'postgresql': {
-            'bin_dir': '',
             'use_slots': True,
             'parameters': CaseInsensitiveDict({p: v[0] for p, v in ConfigHandler.CMDLINE_OPTIONS.items()
-                                               if p not in ('wal_keep_segments', 'wal_keep_size')})
+                                               if v[0] is not None and p not in ('wal_keep_segments', 'wal_keep_size')})
         }
     }
 
     def __init__(self, configfile: str,
                  validator: Optional[Callable[[Dict[str, Any]], List[str]]] = default_validator) -> None:
+        """Create a new instance of :class:`Config` and validate the loaded configuration using *validator*.
+
+        .. note::
+            Patroni will read configuration from these locations in this order:
+
+              * file or directory path passed as command-line argument (*configfile*), if it exists and the file or
+                files found in the directory can be parsed (see :meth:`~Config._load_config_path`), otherwise
+              * YAML file passed via the environment variable (see :attr:`PATRONI_CONFIG_VARIABLE`), if the referenced
+                file exists and can be parsed, otherwise
+              * from configuration values defined as environment variables, see
+                :meth:`~Config._build_environment_configuration`.
+
+        :param configfile: path to Patroni configuration file.
+        :param validator: function used to validate Patroni configuration. It should receive a dictionary which
+            represents Patroni configuration, and return a list of zero or more error messages based on validation.
+
+        :raises:
+            :class:`ConfigParseError`: if any issue is reported by *validator*.
+        """
         self._modify_version = -1
         self._dynamic_configuration = {}
 
         self.__environment_configuration = self._build_environment_configuration()
 
-        # Patroni reads the configuration from the command-line argument if it exists, otherwise from the environment
         self._config_file = configfile if configfile and os.path.exists(configfile) else None
         if self._config_file:
             self._local_configuration = self._load_config_file()
@@ -223,21 +289,48 @@ class Config(object):
         self.__effective_configuration = self._build_effective_configuration({}, self._local_configuration)
         self._data_dir = self.__effective_configuration.get('postgresql', {}).get('data_dir', "")
         self._cache_file = os.path.join(self._data_dir, self.__CACHE_FILENAME)
-        self._load_cache()
+        if validator:  # patronictl uses validator=None and we don't want to load anything from local cache in this case
+            self._load_cache()
         self._cache_needs_saving = False
 
     @property
-    def config_file(self) -> Union[str, None]:
+    def config_file(self) -> Optional[str]:
+        """Path to Patroni configuration file, if any, else ``None``."""
         return self._config_file
 
     @property
     def dynamic_configuration(self) -> Dict[str, Any]:
+        """Deep copy of cached Patroni dynamic configuration."""
         return deepcopy(self._dynamic_configuration)
 
-    def _load_config_path(self, path: str) -> Dict[str, Any]:
+    @property
+    def local_configuration(self) -> Dict[str, Any]:
+        """Deep copy of cached Patroni local configuration.
+
+        :returns: copy of :attr:`~Config._local_configuration`
         """
-        If path is a file, loads the yml file pointed to by path.
+        return deepcopy(dict(self._local_configuration))
-        If path is a directory, loads all yml files in that directory in alphabetical order
+
+    @classmethod
+    def get_default_config(cls) -> Dict[str, Any]:
+        """Deep copy default configuration.
+
+        :returns: copy of :attr:`~Config.__DEFAULT_CONFIG`
+        """
+        return deepcopy(cls.__DEFAULT_CONFIG)
+
+    def _load_config_path(self, path: str) -> Dict[str, Any]:
+        """Load Patroni configuration file(s) from *path*.
+
+        If *path* is a file, load the yml file pointed to by *path*.
+        If *path* is a directory, load all yml files in that directory in alphabetical order.
+
+        :param path: path to either an YAML configuration file, or to a folder containing YAML configuration files.
+
+        :returns: configuration after reading the configuration file(s) from *path*.
+
+        :raises:
+            :class:`ConfigParseError`: if *path* is invalid.
         """
         if os.path.isfile(path):
             files = [path]
@@ -256,14 +349,18 @@ class Config(object):
         return overall_config
 
     def _load_config_file(self) -> Dict[str, Any]:
-        """Loads config.yaml from filesystem and applies some values which were set via ENV"""
+        """Load configuration file(s) from filesystem and apply values which were set via environment variables.
+
+        :returns: final configuration after merging configuration file(s) and environment variables.
+        """
         if TYPE_CHECKING:  # pragma: no cover
-            assert self._config_file is not None
+            assert self.config_file is not None
-        config = self._load_config_path(self._config_file)
+        config = self._load_config_path(self.config_file)
         patch_config(config, self.__environment_configuration)
         return config
 
     def _load_cache(self) -> None:
+        """Load dynamic configuration from ``patroni.dynamic.json``."""
         if os.path.isfile(self._cache_file):
             try:
                 with open(self._cache_file) as f:
@@ -272,14 +369,22 @@ class Config(object):
                 logger.exception('Exception when loading file: %s', self._cache_file)
 
     def save_cache(self) -> None:
+        """Save dynamic configuration to ``patroni.dynamic.json`` under Postgres data directory.
+
+        .. note::
+            ``patroni.dynamic.jsonXXXXXX`` is created as a temporary file and than renamed to ``patroni.dynamic.json``,
+            where ``XXXXXX`` is a random suffix.
+        """
         if self._cache_needs_saving:
             tmpfile = fd = None
             try:
+                pg_perm.set_permissions_from_data_directory(self._data_dir)
                 (fd, tmpfile) = tempfile.mkstemp(prefix=self.__CACHE_FILENAME, dir=self._data_dir)
                 with os.fdopen(fd, 'w') as f:
                     fd = None
                     json.dump(self.dynamic_configuration, f)
                 tmpfile = shutil.move(tmpfile, self._cache_file)
+                os.chmod(self._cache_file, pg_perm.file_create_mode)
                 self._cache_needs_saving = False
             except Exception:
                 logger.exception('Exception when saving file: %s', self._cache_file)
@@ -296,9 +401,16 @@ class Config(object):
 
     # configuration could be either ClusterConfig or dict
     def set_dynamic_configuration(self, configuration: Union[ClusterConfig, Dict[str, Any]]) -> bool:
+        """Set dynamic configuration values with given *configuration*.
+
+        :param configuration: new dynamic configuration values. Supports :class:`dict` for backward compatibility.
+
+        :returns: ``True`` if changes have been detected between current dynamic configuration and the new dynamic
+            *configuration*, ``False`` otherwise.
+        """
         if isinstance(configuration, ClusterConfig):
             if self._modify_version == configuration.modify_version:
-                return False  # If the version didn't changed there is nothing to do
+                return False  # If the version didn't change there is nothing to do
             self._modify_version = configuration.modify_version
             configuration = configuration.data
 
@@ -314,6 +426,14 @@ class Config(object):
         return False
 
     def reload_local_configuration(self) -> Optional[bool]:
+        """Reload configuration values from the configuration file(s).
+
+        .. note::
+            Designed to be used when user applies changes to configuration file(s), so Patroni can use the new values
+            with a reload instead of a restart.
+
+        :returns: ``True`` if changes have been detected between current local configuration
+        """
         if self.config_file:
             try:
                 configuration = self._load_config_file()
@@ -329,12 +449,80 @@ class Config(object):
 
     @staticmethod
     def _process_postgresql_parameters(parameters: Dict[str, Any], is_local: bool = False) -> Dict[str, Any]:
-        return {name: value for name, value in (parameters or {}).items()
+        """Process Postgres *parameters*.
-                if name not in ConfigHandler.CMDLINE_OPTIONS
+
-                or not is_local and ConfigHandler.CMDLINE_OPTIONS[name][1](value)}
+        .. note::
+            If *is_local* configuration discard any setting from *parameters* that is listed under
+            :attr:`~patroni.postgresql.config.ConfigHandler.CMDLINE_OPTIONS` as those are supposed to be set only
+            through dynamic configuration.
+
+            When setting parameters from :attr:`~patroni.postgresql.config.ConfigHandler.CMDLINE_OPTIONS` through
+            dynamic configuration their value will be validated as per the validator defined in that very same
+            attribute entry. If the given value cannot be validated, a warning will be logged and the default value of
+            the GUC will be used instead.
+
+            Some parameters from :attr:`~patroni.postgresql.config.ConfigHandler.CMDLINE_OPTIONS` cannot be set even if
+            not *is_local* configuration:
+
+                * ``listen_addresses``: inferred from ``postgresql.listen`` local configuration or from
+                    ``PATRONI_POSTGRESQL_LISTEN`` environment variable;
+                * ``port``: inferred from ``postgresql.listen`` local configuration or from
+                    ``PATRONI_POSTGRESQL_LISTEN`` environment variable;
+                * ``cluster_name``: set through ``scope`` local configuration or through ``PATRONI_SCOPE`` environment
+                    variable;
+                * ``hot_standby``: always enabled;
+                * ``wal_log_hints``: always enabled.
+
+        :param parameters: Postgres parameters to be processed. Should be the parsed YAML value of
+            ``postgresql.parameters`` configuration, either from local or from dynamic configuration.
+
+        :param is_local: should be ``True`` if *parameters* refers to local configuration, or ``False`` if *parameters*
+            refers to dynamic configuration.
+
+        :returns: new value for ``postgresql.parameters`` after processing and validating *parameters*.
+        """
+        pg_params: Dict[str, Any] = {}
+
+        for name, value in (parameters or {}).items():
+            if name not in ConfigHandler.CMDLINE_OPTIONS:
+                pg_params[name] = value
+            elif not is_local:
+                if ConfigHandler.CMDLINE_OPTIONS[name][1](value):
+                    pg_params[name] = value
+                else:
+                    logger.warning("postgresql parameter %s=%s failed validation, defaulting to %s",
+                                   name, value, ConfigHandler.CMDLINE_OPTIONS[name][0])
+
+        return pg_params
 
     def _safe_copy_dynamic_configuration(self, dynamic_configuration: Dict[str, Any]) -> Dict[str, Any]:
-        config = deepcopy(self.__DEFAULT_CONFIG)
+        """Create a copy of *dynamic_configuration*.
+
+        Merge *dynamic_configuration* with :attr:`__DEFAULT_CONFIG` (*dynamic_configuration* takes precedence), and
+        process ``postgresql.parameters`` from *dynamic_configuration* through :func:`_process_postgresql_parameters`,
+        if present.
+
+        .. note::
+            The following settings are not allowed in ``postgresql`` section as they are intended to be local
+            configuration, and are removed if present:
+
+                * ``connect_address``;
+                * ``proxy_address``;
+                * ``listen``;
+                * ``config_dir``;
+                * ``data_dir``;
+                * ``pgpass``;
+                * ``authentication``;
+
+            Besides that any setting present in *dynamic_configuration* but absent from :attr:`__DEFAULT_CONFIG` is
+            discarded.
+
+        :param dynamic_configuration: Patroni dynamic configuration.
+
+        :returns: copy of *dynamic_configuration*, merged with default dynamic configuration and with some sanity checks
+            performed over it.
+        """
+        config = self.get_default_config()
 
         for name, value in dynamic_configuration.items():
             if name == 'postgresql':
@@ -354,9 +542,25 @@ class Config(object):
 
     @staticmethod
     def _build_environment_configuration() -> Dict[str, Any]:
+        """Get local configuration settings that were specified through environment variables.
+
+        :returns: dictionary containing the found environment variables and their values, respecting the expected
+            structure of Patroni configuration.
+        """
         ret: Dict[str, Any] = defaultdict(dict)
 
-        def _popenv(name: str) -> Union[str, None]:
+        def _popenv(name: str) -> Optional[str]:
+            """Get value of environment variable *name*.
+
+            .. note::
+                *name* is prefixed with :data:`~patroni.PATRONI_ENV_PREFIX` when searching in the environment.
+
+                Also, the corresponding environment variable is removed from the environment upon reading its value.
+
+            :param name: name of the environment variable.
+
+            :returns: value of *name*, if present in the environment, otherwise ``None``.
+            """
             return os.environ.pop(PATRONI_ENV_PREFIX + name.upper(), None)
 
         for param in ('name', 'namespace', 'scope'):
@@ -365,6 +569,23 @@ class Config(object):
                 ret[param] = value
 
         def _fix_log_env(name: str, oldname: str) -> None:
+            """Normalize a log related environment variable.
+
+            .. note::
+                Patroni used to support different names for log related environment variables in the past. As the
+                environment variables were renamed, this function takes care of mapping and normalizing the environment.
+
+                *name* is prefixed with :data:`~patroni.PATRONI_ENV_PREFIX` and ``LOG`` when searching in the
+                environment.
+
+                *oldname* is prefixed with :data:`~patroni.PATRONI_ENV_PREFIX` when searching in the environment.
+
+                If both *name* and *oldname* are set in the environment, *name* takes precedence.
+
+            :param name: new name of a log related environment variable.
+            :param oldname: original name of a log related environment variable.
+            :type oldname: str
+            """
             value = _popenv(oldname)
             name = PATRONI_ENV_PREFIX + 'LOG_' + name.upper()
             if value and name not in os.environ:
@@ -374,6 +595,15 @@ class Config(object):
             _fix_log_env(name, oldname)
 
         def _set_section_values(section: str, params: List[str]) -> None:
+            """Get value of *params* environment variables that are related with *section*.
+
+            .. note::
+                The values are retrieved from the environment and updated directly into the returning dictionary of
+                :func:`_build_environment_configuration`.
+
+            :param section: configuration section the *params* belong to.
+            :param params: name of the Patroni settings.
+            """
             for param in params:
                 value = _popenv(section + '_' + param)
                 if value:
@@ -395,6 +625,7 @@ class Config(object):
             if value:
                 ret['postgresql'].setdefault('bin_name', {})[binary] = value
 
+        # parse all values retrieved from the environment as Python objects, according to the expected type
         for first, second in (('restapi', 'allowlist_include_members'), ('ctl', 'insecure')):
             value = ret.get(first, {}).pop(second, None)
             if value:
@@ -411,7 +642,13 @@ class Config(object):
                     if value is not None:
                         ret[first][second] = value
 
-        def _parse_list(value: str) -> Union[List[str], None]:
+        def _parse_list(value: str) -> Optional[List[str]]:
+            """Parse an YAML list *value* as a :class:`list`.
+
+            :param value: YAML list as a string.
+
+            :returns: *value* as :class:`list`.
+            """
             if not (value.strip().startswith('-') or '[' in value):
                 value = '[{0}]'.format(value)
             try:
@@ -427,7 +664,13 @@ class Config(object):
                 if value:
                     ret[first][second] = value
 
-        def _parse_dict(value: str) -> Union[Dict[str, Any], None]:
+        def _parse_dict(value: str) -> Optional[Dict[str, Any]]:
+            """Parse an YAML dictionary *value* as a :class:`dict`.
+
+            :param value: YAML dictionary as a string.
+
+            :returns: *value* as :class:`dict`.
+            """
             if not value.strip().startswith('{'):
                 value = '{{{0}}}'.format(value)
             try:
@@ -444,17 +687,25 @@ class Config(object):
                     if value:
                         ret[first][second] = value
 
-        def _get_auth(name: str, params: Optional[Collection[str]] = None) -> Dict[str, str]:
+        def _get_auth(name: str, params: Collection[str] = _AUTH_ALLOWED_PARAMETERS[:2]) -> Dict[str, str]:
+            """Get authorization related environment variables *params* from section *name*.
+
+            :param name: name of a configuration section that may contain authorization *params*.
+            :param params: the authorization settings that may be set under section *name*.
+
+            :returns: dictionary containing environment values for authorization *params* of section *name*.
+            """
             ret: Dict[str, str] = {}
-            for param in params or _AUTH_ALLOWED_PARAMETERS[:2]:
+            for param in params:
                 value = _popenv(name + '_' + param)
                 if value:
                     ret[param] = value
             return ret
 
-        restapi_auth = _get_auth('restapi')
+        for section in ('ctl', 'restapi'):
-        if restapi_auth:
+            auth = _get_auth(section)
-            ret['restapi']['authentication'] = restapi_auth
+            if auth:
+                ret[section]['authentication'] = auth
 
         authentication = {}
         for user_type in ('replication', 'superuser', 'rewind'):
@@ -468,13 +719,14 @@ class Config(object):
         for param in list(os.environ.keys()):
             if param.startswith(PATRONI_ENV_PREFIX):
                 # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
-                name, suffix = (param[8:].split('_', 1) + [''])[:2]
+                name, suffix = (param[len(PATRONI_ENV_PREFIX):].split('_', 1) + [''])[:2]
                 if suffix in ('HOST', 'HOSTS', 'PORT', 'USE_PROXIES', 'PROTOCOL', 'SRV', 'SRV_SUFFIX', 'URL', 'PROXY',
                               'CACERT', 'CERT', 'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC', 'CONSISTENCY',
                               'REGISTER_SERVICE', 'SERVICE_CHECK_INTERVAL', 'SERVICE_CHECK_TLS_SERVER_NAME',
                               'SERVICE_TAGS', 'NAMESPACE', 'CONTEXT', 'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL',
                               'POD_IP', 'PORTS', 'LABELS', 'BYPASS_API_SERVICE', 'RETRIABLE_HTTP_CODES', 'KEY_PASSWORD',
-                              'USE_SSL', 'SET_ACLS', 'GROUP', 'DATABASE') and name:
+                              'USE_SSL', 'SET_ACLS', 'GROUP', 'DATABASE', 'LEADER_LABEL_VALUE', 'FOLLOWER_LABEL_VALUE',
+                              'STANDBY_LEADER_LABEL_VALUE', 'TMP_ROLE_LABEL') and name:
                     value = os.environ.pop(param)
                     if name == 'CITUS':
                         if suffix == 'GROUP':
@@ -498,14 +750,14 @@ class Config(object):
         users = {}
         for param in list(os.environ.keys()):
             if param.startswith(PATRONI_ENV_PREFIX):
-                name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
+                name, suffix = (param[len(PATRONI_ENV_PREFIX):].rsplit('_', 1) + [''])[:2]
                 # PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>
                 # CREATE USER "<username>" WITH <OPTIONS> PASSWORD '<password>'
                 if name and suffix == 'PASSWORD':
                     password = os.environ.pop(param)
                     if password:
                         users[name] = {'password': password}
-                        options = os.environ.pop(param[:-9] + '_OPTIONS', None)
+                        options = os.environ.pop(param[:-9] + '_OPTIONS', None)  # replace "_PASSWORD" with "_OPTIONS"
                         options = options and _parse_list(options)
                         if options:
                             users[name]['options'] = options
@@ -516,6 +768,16 @@ class Config(object):
 
     def _build_effective_configuration(self, dynamic_configuration: Dict[str, Any],
                                        local_configuration: Dict[str, Union[Dict[str, Any], Any]]) -> Dict[str, Any]:
+        """Build effective configuration by merging *dynamic_configuration* and *local_configuration*.
+
+        .. note::
+            *local_configuration* takes precedence over *dynamic_configuration* if a setting is defined in both.
+
+        :param dynamic_configuration: Patroni dynamic configuration.
+        :param local_configuration: Patroni local configuration.
+
+        :returns: _description_
+        """
         config = self._safe_copy_dynamic_configuration(dynamic_configuration)
         for name, value in local_configuration.items():
             if name == 'citus':  # remove invalid citus configuration
@@ -531,9 +793,10 @@ class Config(object):
             elif name not in config or name in ['watchdog']:
                 config[name] = deepcopy(value) if value else {}
 
-        # restapi server expects to get restapi.auth = 'username:password'
+        # restapi server expects to get restapi.auth = 'username:password' and similarly for `ctl`
-        if 'restapi' in config and 'authentication' in config['restapi']:
+        for section in ('ctl', 'restapi'):
-            config['restapi']['auth'] = '{username}:{password}'.format(**config['restapi']['authentication'])
+            if section in config and 'authentication' in config[section]:
+                config[section]['auth'] = '{username}:{password}'.format(**config[section]['authentication'])
 
         # special treatment for old config
 
@@ -578,23 +841,57 @@ class Config(object):
         return config
 
     def get(self, key: str, default: Optional[Any] = None) -> Any:
+        """Get effective value of ``key`` setting from Patroni configuration root.
+
+        Designed to work the same way as :func:`dict.get`.
+
+        :param key: name of the setting.
+        :param default: default value if *key* is not present in the effective configuration.
+
+        :returns: value of *key*, if present in the effective configuration, otherwise *default*.
+        """
         return self.__effective_configuration.get(key, default)
 
     def __contains__(self, key: str) -> bool:
+        """Check if setting *key* is present in the effective configuration.
+
+        Designed to work the same way as :func:`dict.__contains__`.
+
+        :param key: name of the setting to be checked.
+
+        :returns: ``True`` if setting *key* exists in effective configuration, else ``False``.
+        """
         return key in self.__effective_configuration
 
     def __getitem__(self, key: str) -> Any:
+        """Get value of setting *key* from effective configuration.
+
+        Designed to work the same way as :func:`dict.__getitem__`.
+
+        :param key: name of the setting.
+
+        :returns: value of setting *key*.
+
+        :raises:
+            :class:`KeyError`: if *key* is not present in effective configuration.
+        """
         return self.__effective_configuration[key]
 
     def copy(self) -> Dict[str, Any]:
+        """Get a deep copy of effective Patroni configuration.
+
+        :returns: a deep copy of the Patroni configuration.
+        """
         return deepcopy(self.__effective_configuration)
 
-    def get_global_config(self, cluster: Union[Cluster, None]) -> GlobalConfig:
+    def get_global_config(self, cluster: Optional[Cluster]) -> GlobalConfig:
         """Instantiate :class:`GlobalConfig` based on input.
 
         Use the configuration from provided *cluster* (the most up-to-date) or from the
         local cache if *cluster.config* is not initialized or doesn't have a valid config.
-        :param cluster: the currently known cluster state from DCS
+
-        :returns: :class:`GlobalConfig` object
+        :param cluster: the currently known cluster state from DCS.
+
+        :returns: :class:`GlobalConfig` object.
         """
         return get_global_config(cluster, self._dynamic_configuration)

patroni/config_generator.py

@@ -0,0 +1,463 @@
+"""patroni ``--generate-config`` machinery."""
+import abc
+import logging
+import os
+import psutil
+import socket
+import sys
+import yaml
+
+from getpass import getuser, getpass
+from contextlib import contextmanager
+from typing import Any, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union
+if TYPE_CHECKING:  # pragma: no cover
+    from psycopg import Cursor
+    from psycopg2 import cursor
+
+from . import psycopg
+from .config import Config
+from .exceptions import PatroniException
+from .postgresql.config import ConfigHandler, parse_dsn
+from .postgresql.misc import postgres_major_version_to_int
+from .utils import get_major_version, parse_bool, patch_config, read_stripped
+
+
+# Mapping between the libpq connection parameters and the environment variables.
+# This dict should be kept in sync with `patroni.utils._AUTH_ALLOWED_PARAMETERS`
+# (we use "username" in the Patroni config for some reason, other parameter names are the same).
+_AUTH_ALLOWED_PARAMETERS_MAPPING = {
+    'user': 'PGUSER',
+    'password': 'PGPASSWORD',
+    'sslmode': 'PGSSLMODE',
+    'sslcert': 'PGSSLCERT',
+    'sslkey': 'PGSSLKEY',
+    'sslpassword': '',
+    'sslrootcert': 'PGSSLROOTCERT',
+    'sslcrl': 'PGSSLCRL',
+    'sslcrldir': 'PGSSLCRLDIR',
+    'gssencmode': 'PGGSSENCMODE',
+    'channel_binding': 'PGCHANNELBINDING'
+}
+_NO_VALUE_MSG = '#FIXME'
+
+
+def get_address() -> Tuple[str, str]:
+    """Try to get hostname and the ip address for it returned by :func:`~socket.gethostname`.
+
+    .. note::
+        Can also return local ip.
+
+    :returns: tuple consisting of the hostname returned by :func:`~socket.gethostname`
+        and the first element in the sorted list of the addresses returned by :func:`~socket.getaddrinfo`.
+        Sorting guarantees it will prefer IPv4.
+        If an exception occured, hostname and ip values are equal to :data:`~patroni.config_generator._NO_VALUE_MSG`.
+    """
+    hostname = None
+    try:
+        hostname = socket.gethostname()
+        return hostname, sorted(socket.getaddrinfo(hostname, 0, socket.AF_UNSPEC, socket.SOCK_STREAM, 0),
+                                key=lambda x: x[0])[0][4][0]
+    except Exception as err:
+        logging.warning('Failed to obtain address: %r', err)
+        return _NO_VALUE_MSG, _NO_VALUE_MSG
+
+
+class AbstractConfigGenerator(abc.ABC):
+    """Object representing the generated Patroni config.
+
+    :ivar output_file: full path to the output file to be used.
+    :ivar pg_major: integer representation of the major PostgreSQL version.
+    :ivar config: dictionary used for the generated configuration storage.
+    """
+
+    _HOSTNAME, _IP = get_address()
+
+    def __init__(self, output_file: Optional[str]) -> None:
+        """Set up the output file (if passed), helper vars and the minimal config structure.
+
+        :param output_file: full path to the output file to be used.
+        """
+        self.output_file = output_file
+        self.pg_major = 0
+        self.config = self.get_template_config()
+
+        self.generate()
+
+    @classmethod
+    def get_template_config(cls) -> Dict[str, Any]:
+        """Generate a template config for further extension (e.g. in the inherited classes).
+
+        :returns: dictionary with the values gathered from Patroni env, hopefully defined hostname and ip address
+                  (otherwise set to :data:`~patroni.config_generator._NO_VALUE_MSG`), and some sane defaults.
+        """
+        template_config: Dict[str, Any] = {
+            'scope': _NO_VALUE_MSG,
+            'name': cls._HOSTNAME,
+            'postgresql': {
+                'data_dir': _NO_VALUE_MSG,
+                'connect_address': _NO_VALUE_MSG + ':5432',
+                'listen': _NO_VALUE_MSG + ':5432',
+                'bin_dir': '',
+                'authentication': {
+                    'superuser': {
+                        'username': 'postgres',
+                        'password': _NO_VALUE_MSG
+                    },
+                    'replication': {
+                        'username': 'replicator',
+                        'password': _NO_VALUE_MSG
+                    }
+                }
+            },
+            'restapi': {
+                'connect_address': cls._IP + ':8008',
+                'listen': cls._IP + ':8008'
+            }
+        }
+
+        dynamic_config = Config.get_default_config()
+        # to properly dump CaseInsensitiveDict as YAML later
+        dynamic_config['postgresql']['parameters'] = dict(dynamic_config['postgresql']['parameters'])
+        config = Config('', None).local_configuration  # Get values from env
+        config.setdefault('bootstrap', {})['dcs'] = dynamic_config
+        config.setdefault('postgresql', {})
+        del config['bootstrap']['dcs']['standby_cluster']
+
+        patch_config(template_config, config)
+        return template_config
+
+    @abc.abstractmethod
+    def generate(self) -> None:
+        """Generate config and store in :attr:`~AbstractConfigGenerator.config`."""
+
+    def write_config(self) -> None:
+        """Write current :attr:`~AbstractConfigGenerator.config` to the output file if provided, to stdout otherwise."""
+        if self.output_file:
+            dir_path = os.path.dirname(self.output_file)
+            if dir_path and not os.path.isdir(dir_path):
+                os.makedirs(dir_path)
+            with open(self.output_file, 'w', encoding='UTF-8') as output_file:
+                yaml.safe_dump(self.config, output_file, default_flow_style=False, allow_unicode=True)
+        else:
+            yaml.safe_dump(self.config, sys.stdout, default_flow_style=False, allow_unicode=True)
+
+
+class SampleConfigGenerator(AbstractConfigGenerator):
+    """Object representing the generated sample Patroni config.
+
+    Sane defults are used based on the gathered PG version.
+    """
+
+    @property
+    def get_auth_method(self) -> str:
+        """Return the preferred authentication method for a specific PG version if provided or the default ``md5``.
+
+        :returns: :class:`str` value for the preferred authentication method.
+        """
+        return 'scram-sha-256' if self.pg_major and self.pg_major >= 100000 else 'md5'
+
+    def _get_int_major_version(self) -> int:
+        """Get major PostgreSQL version from the binary as an integer.
+
+        :returns: an integer PostgreSQL major version representation gathered from the PostgreSQL binary.
+                  See :func:`~patroni.postgresql.misc.postgres_major_version_to_int` and
+                  :func:`~patroni.utils.get_major_version`.
+        """
+        postgres_bin = ((self.config.get('postgresql') or {}).get('bin_name') or {}).get('postgres', 'postgres')
+        return postgres_major_version_to_int(get_major_version(self.config['postgresql'].get('bin_dir'), postgres_bin))
+
+    def generate(self) -> None:
+        """Generate sample config using some sane defaults and update :attr:`~AbstractConfigGenerator.config`."""
+        self.pg_major = self._get_int_major_version()
+
+        self.config['postgresql']['parameters'] = {'password_encryption': self.get_auth_method}
+        username = self.config["postgresql"]["authentication"]["replication"]["username"]
+        self.config['postgresql']['pg_hba'] = [
+            f'host all all all {self.get_auth_method}',
+            f'host replication {username} all {self.get_auth_method}'
+        ]
+
+        # add version-specific configuration
+        wal_keep_param = 'wal_keep_segments' if self.pg_major < 130000 else 'wal_keep_size'
+        self.config['bootstrap']['dcs']['postgresql']['parameters'][wal_keep_param] = \
+            ConfigHandler.CMDLINE_OPTIONS[wal_keep_param][0]
+
+        self.config['bootstrap']['dcs']['postgresql']['use_pg_rewind'] = True
+        if self.pg_major >= 110000:
+            self.config['postgresql']['authentication'].setdefault(
+                'rewind', {'username': 'rewind_user'}).setdefault('password', _NO_VALUE_MSG)
+
+
+class RunningClusterConfigGenerator(AbstractConfigGenerator):
+    """Object representing the Patroni config generated using information gathered from the running instance.
+
+    :ivar dsn: DSN string for the local instance to get GUC values from (if provided).
+    :ivar parsed_dsn: DSN string parsed into a dictionary (see :func:`~patroni.postgresql.config.parse_dsn`).
+    """
+
+    def __init__(self, output_file: Optional[str] = None, dsn: Optional[str] = None) -> None:
+        """Additionally store the passed dsn (if any) in both original and parsed version and run config generation.
+
+        :param output_file: full path to the output file to be used.
+        :param dsn: DSN string for the local instance to get GUC values from.
+
+        :raises:
+            :exc:`~patroni.exceptions.PatroniException`: if DSN parsing failed.
+        """
+        self.dsn = dsn
+        self.parsed_dsn = {}
+
+        super().__init__(output_file)
+
+    @property
+    def _get_hba_conn_types(self) -> Tuple[str, ...]:
+        """Return the connection types allowed.
+
+        If :attr:`~RunningClusterConfigGenerator.pg_major` is defined, adds additional parameters
+        for PostgreSQL version >=16.
+
+        :returns: tuple of the connection methods allowed.
+        """
+        allowed_types = ('local', 'host', 'hostssl', 'hostnossl', 'hostgssenc', 'hostnogssenc')
+        if self.pg_major and self.pg_major >= 160000:
+            allowed_types += ('include', 'include_if_exists', 'include_dir')
+        return allowed_types
+
+    @property
+    def _required_pg_params(self) -> List[str]:
+        """PG configuration prameters that have to be always present in the generated config.
+
+        :returns: list of the parameter names.
+        """
+        return ['hba_file', 'ident_file', 'config_file', 'data_directory'] + \
+            list(ConfigHandler.CMDLINE_OPTIONS.keys())
+
+    def _get_bin_dir_from_running_instance(self) -> str:
+        """Define the directory postgres binaries reside using postmaster's pid executable.
+
+        :returns: path to the PostgreSQL binaries directory.
+
+        :raises:
+            :exc:`~patroni.exceptions.PatroniException`: if:
+
+                * pid could not be obtained from the ``postmaster.pid`` file; or
+                * :exc:`OSError` occured during ``postmaster.pid`` file handling; or
+                * the obtained postmaster pid doesn't exist.
+        """
+        postmaster_pid = None
+        data_dir = self.config['postgresql']['data_dir']
+        try:
+            with open(f"{data_dir}/postmaster.pid", 'r') as pid_file:
+                postmaster_pid = pid_file.readline()
+                if not postmaster_pid:
+                    raise PatroniException('Failed to obtain postmaster pid from postmaster.pid file')
+                postmaster_pid = int(postmaster_pid.strip())
+        except OSError as err:
+            raise PatroniException(f'Error while reading postmaster.pid file: {err}')
+        try:
+            return os.path.dirname(psutil.Process(postmaster_pid).exe())
+        except psutil.NoSuchProcess:
+            raise PatroniException("Obtained postmaster pid doesn't exist.")
+
+    @contextmanager
+    def _get_connection_cursor(self) -> Iterator[Union['cursor', 'Cursor[Any]']]:
+        """Get cursor for the PG connection established based on the stored information.
+
+        :raises:
+            :exc:`~patroni.exceptions.PatroniException`: if :exc:`psycopg.Error` occured.
+        """
+        try:
+            conn = psycopg.connect(dsn=self.dsn,
+                                   password=self.config['postgresql']['authentication']['superuser']['password'])
+            with conn.cursor() as cur:
+                yield cur
+            conn.close()
+        except psycopg.Error as e:
+            raise PatroniException(f'Failed to establish PostgreSQL connection: {e}')
+
+    def _set_pg_params(self, cur: Union['cursor', 'Cursor[Any]']) -> None:
+        """Extend :attr:`~RunningClusterConfigGenerator.config` with the actual PG GUCs values.
+
+        THe following GUC values are set:
+
+            * Non-internal having configuration file, postmaster command line or environment variable
+              as a source.
+
+            * List of the always required parameters (see :meth:`~RunningClusterConfigGenerator._required_pg_params`).
+
+        :param cur: connection cursor to use.
+        """
+        cur.execute("SELECT name, current_setting(name) FROM pg_settings "
+                    "WHERE context <> 'internal' "
+                    "AND source IN ('configuration file', 'command line', 'environment variable') "
+                    "AND category <> 'Write-Ahead Log / Recovery Target' "
+                    "AND setting <> '(disabled)' "
+                    "OR name = ANY(%s)", (self._required_pg_params,))
+
+        helper_dict = dict.fromkeys(['port', 'listen_addresses'])
+        self.config['postgresql'].setdefault('parameters', {})
+        for param, value in cur.fetchall():
+            if param == 'data_directory':
+                self.config['postgresql']['data_dir'] = value
+            elif param == 'cluster_name' and value:
+                self.config['scope'] = value
+            elif param in ('archive_command', 'restore_command',
+                           'archive_cleanup_command', 'recovery_end_command',
+                           'ssl_passphrase_command', 'hba_file',
+                           'ident_file', 'config_file'):
+                # write commands to the local config due to security implications
+                # write hba/ident/config_file to local config to ensure they are not removed later
+                self.config['postgresql']['parameters'][param] = value
+            elif param in helper_dict:
+                helper_dict[param] = value
+            else:
+                self.config['bootstrap']['dcs']['postgresql']['parameters'][param] = value
+
+        connect_port = self.parsed_dsn.get('port', os.getenv('PGPORT', helper_dict['port']))
+        self.config['postgresql']['connect_address'] = f'{self._IP}:{connect_port}'
+        self.config['postgresql']['listen'] = f'{helper_dict["listen_addresses"]}:{helper_dict["port"]}'
+
+    def _set_su_params(self) -> None:
+        """Extend :attr:`~RunningClusterConfigGenerator.config` with the superuser auth information.
+
+        Information set is based on the options used for connection.
+        """
+        su_params: Dict[str, str] = {}
+        for conn_param, env_var in _AUTH_ALLOWED_PARAMETERS_MAPPING.items():
+            val = self.parsed_dsn.get(conn_param, os.getenv(env_var))
+            if val:
+                su_params[conn_param] = val
+        patroni_env_su_username = ((self.config.get('authentication') or {}).get('superuser') or {}).get('username')
+        patroni_env_su_pwd = ((self.config.get('authentication') or {}).get('superuser') or {}).get('password')
+        # because we use "username" in the config for some reason
+        su_params['username'] = su_params.pop('user', patroni_env_su_username) or getuser()
+        su_params['password'] = su_params.get('password', patroni_env_su_pwd) or \
+            getpass('Please enter the user password:')
+        self.config['postgresql']['authentication'] = {
+            'superuser': su_params,
+            'replication': {'username': _NO_VALUE_MSG, 'password': _NO_VALUE_MSG}
+        }
+
+    def _set_conf_files(self) -> None:
+        """Extend :attr:`~RunningClusterConfigGenerator.config` with ``pg_hba.conf`` and ``pg_ident.conf`` content.
+
+        .. note::
+            This function only defines ``postgresql.pg_hba`` and ``postgresql.pg_ident`` when
+            ``hba_file`` and ``ident_file`` are set to the defaults. It may happen these files
+            are located outside of ``PGDATA`` and Patroni doesn't have write permissions for them.
+
+        :raises:
+            :exc:`~patroni.exceptions.PatroniException`: if :exc:`OSError` occured during the conf files handling.
+        """
+        default_hba_path = os.path.join(self.config['postgresql']['data_dir'], 'pg_hba.conf')
+        if self.config['postgresql']['parameters']['hba_file'] == default_hba_path:
+            try:
+                self.config['postgresql']['pg_hba'] = list(
+                    filter(lambda i: i and i.split()[0] in self._get_hba_conn_types, read_stripped(default_hba_path)))
+            except OSError as err:
+                raise PatroniException(f'Failed to read pg_hba.conf: {err}')
+
+        default_ident_path = os.path.join(self.config['postgresql']['data_dir'], 'pg_ident.conf')
+        if self.config['postgresql']['parameters']['ident_file'] == default_ident_path:
+            try:
+                self.config['postgresql']['pg_ident'] = [i for i in read_stripped(default_ident_path)
+                                                         if i and not i.startswith('#')]
+            except OSError as err:
+                raise PatroniException(f'Failed to read pg_ident.conf: {err}')
+            if not self.config['postgresql']['pg_ident']:
+                del self.config['postgresql']['pg_ident']
+
+    def _enrich_config_from_running_instance(self) -> None:
+        """Extend :attr:`~RunningClusterConfigGenerator.config` with the values gathered from the running instance.
+
+        Retrieve the following information from the running PostgreSQL instance:
+
+        * superuser auth parameters (see :meth:`~RunningClusterConfigGenerator._set_su_params`);
+        * some GUC values (see :meth:`~RunningClusterConfigGenerator._set_pg_params`);
+        * ``postgresql.connect_address``, ``postgresql.listen``;
+        * ``postgresql.pg_hba`` and ``postgresql.pg_ident`` (see :meth:`~RunningClusterConfigGenerator._set_conf_files`)
+
+        And redefine ``scope`` with the ``cluster_name`` GUC value if set.
+
+        :raises:
+            :exc:`~patroni.exceptions.PatroniException`: if the provided user doesn't have superuser privileges.
+        """
+        self._set_su_params()
+
+        with self._get_connection_cursor() as cur:
+            self.pg_major = getattr(cur.connection, 'server_version', 0)
+
+            if not parse_bool(cur.connection.info.parameter_status('is_superuser')):
+                raise PatroniException('The provided user does not have superuser privilege')
+
+            self._set_pg_params(cur)
+
+        self._set_conf_files()
+
+    def generate(self) -> None:
+        """Generate config using the info gathered from the specified running PG instance.
+
+        Result is written to :attr:`~RunningClusterConfigGenerator.config`.
+        """
+        if self.dsn:
+            self.parsed_dsn = parse_dsn(self.dsn) or {}
+            if not self.parsed_dsn:
+                raise PatroniException('Failed to parse DSN string')
+
+        self._enrich_config_from_running_instance()
+        self.config['postgresql']['bin_dir'] = self._get_bin_dir_from_running_instance()
+
+
+def generate_config(output_file: str, sample: bool, dsn: Optional[str]) -> None:
+    """Generate Patroni configuration file.
+
+    Gather all the available non-internal GUC values having configuration file, postmaster command line or environment
+    variable as a source and store them in the appropriate part of Patroni configuration (``postgresql.parameters`` or
+    ``bootstrap.dcs.postgresql.parameters``). Either the provided DSN (takes precedence) or PG ENV vars will be used
+    for the connection. If password is not provided, it should be entered via prompt.
+
+    The created configuration contains:
+    * ``scope``: ``cluster_name`` GUC value or ``PATRONI_SCOPE ENV`` variable value if available.
+    * ``name``: ``PATRONI_NAME`` ENV variable value if set, otherwise hostname.
+
+    * ``bootstrap.dcs``: section with all the parameters (incl. the majority of PG GUCs) set to their default values
+      defined by Patroni and adjusted by the source instances's configuration values.
+
+    * ``postgresql.parameters``: the source instance's ``archive_command``, ``restore_command``,
+      ``archive_cleanup_command``, ``recovery_end_command``, ``ssl_passphrase_command``, ``hba_file``, ``ident_file``,
+      ``config_file`` GUC values.
+
+    * ``postgresql.bin_dir``: path to Postgres binaries gathered from the running instance or, if not available,
+      the value of ``PATRONI_POSTGRESQL_BIN_DIR`` ENV variable. Otherwise, an empty string.
+
+    * ``postgresql.datadir``: the value gathered from the corresponding PG GUC.
+    * ``postgresql.listen``: source instance's ``listen_addresses`` and port GUC values.
+    * ``postgresql.connect_address``: if possible, generated from the connection params.
+    * ``postgresql.authentication``:
+
+        * superuser and replication users defined (if possible, usernames are set from the respective Patroni ENV vars,
+          otherwise the default ``postgres`` and ``replicator`` values are used).
+          If not a sample config, either DSN or PG ENV vars are used to define superuser authentication parameters.
+
+        * rewind user is defined only for sample config, if PG version can be defined and PG version is >=11
+          (if possible, username is set from the respective Patroni ENV var).
+
+    * ``bootstrap.dcs.postgresql.use_pg_rewind`` set to ``True`` for a sample config only.
+    * ``postgresql.pg_hba`` defaults or the lines gathered from the source instance's ``hba_file``.
+    * ``postgresql.pg_ident`` the lines gathered from the source instance's ``ident_file``.
+
+    :param output_file: Full path to the configuration file to be used. If not provided, result is sent to ``stdout``.
+    :param sample: Optional flag. If set, no source instance will be used - generate config with some sane defaults.
+    :param dsn: Optional DSN string for the local instance to get GUC values from.
+    """
+    try:
+        if sample:
+            config_generator = SampleConfigGenerator(output_file)
+        else:
+            config_generator = RunningClusterConfigGenerator(output_file, dsn)
+
+        config_generator.write_config()
+    except PatroniException as e:
+        sys.exit(str(e))
+    except Exception as e:
+        sys.exit(f'Unexpected exception: {e}')

patroni/ctl.py

@@ -16,8 +16,6 @@ import click
 import codecs
 import copy
 import datetime
-import dateutil.parser
-import dateutil.tz
 import difflib
 import io
 import json
@@ -36,7 +34,7 @@ from collections import defaultdict
 from contextlib import contextmanager
 from prettytable import ALL, FRAME, PrettyTable
 from urllib.parse import urlparse
-from typing import Any, Dict, Generator, Iterator, List, Optional, Union, Tuple, TYPE_CHECKING
+from typing import Any, Dict, Iterator, List, Optional, Union, Tuple, TYPE_CHECKING
 if TYPE_CHECKING:  # pragma: no cover
     from psycopg import Cursor
     from psycopg2 import cursor
@@ -46,10 +44,12 @@ try:
 except ImportError:  # pragma: no cover
     from cdiff import markup_to_pager, PatchStream  # pyright: ignore [reportMissingModuleSource]
 
+from .config import Config, get_global_config
 from .dcs import get_dcs as _get_dcs, AbstractDCS, Cluster, Member
 from .exceptions import PatroniException
+from .manual_failover import ManualFailover
 from .postgresql.misc import postgres_version_to_int
-from .utils import cluster_as_json, patch_config, polling_loop
+from .utils import cluster_as_json, parse_schedule, patch_config, polling_loop
 from .request import PatroniRequest
 from .version import __version__
 
@@ -225,8 +225,6 @@ def load_config(path: str, dcs_url: Optional[str]) -> Dict[str, Any]:
     :raises:
         :class:`PatroniCtlException`: if *path* does not exist or is not readable.
     """
-    from patroni.config import Config
-
     if not (os.path.exists(path) and os.access(path, os.R_OK)):
         if path != CONFIG_FILE_PATH:    # bail if non-default config location specified but file not found / readable
             raise PatroniCtlException('Provided config file {0} not existing or no read rights.'
@@ -254,7 +252,6 @@ arg_cluster_name = click.argument('cluster_name', required=False,
 option_default_citus_group = click.option('--group', required=False, type=int, help='Citus group',
                                           default=lambda: click.get_current_context().obj.get('citus', {}).get('group'))
 option_citus_group = click.option('--group', required=False, type=int, help='Citus group')
-option_insecure = click.option('-k', '--insecure', is_flag=True, help='Allow connections to SSL sites without certs')
 role_choice = click.Choice(['leader', 'primary', 'standby-leader', 'replica', 'standby', 'any', 'master'])
 
 
@@ -262,10 +259,12 @@ role_choice = click.Choice(['leader', 'primary', 'standby-leader', 'replica', 's
 @click.option('--config-file', '-c', help='Configuration file',
               envvar='PATRONICTL_CONFIG_FILE', default=CONFIG_FILE_PATH)
 @click.option('--dcs-url', '--dcs', '-d', 'dcs_url', help='The DCS connect url', envvar='DCS_URL')
-@option_insecure
+@click.option('-k', '--insecure', is_flag=True, help='Allow connections to SSL sites without certs')
 @click.pass_context
 def ctl(ctx: click.Context, config_file: str, dcs_url: Optional[str], insecure: bool) -> None:
-    """Entry point of ``patronictl`` utility.
+    """Command-line interface for interacting with Patroni.
+    \f
+    Entry point of ``patronictl`` utility.
 
     Load the configuration file.
 
@@ -645,7 +644,8 @@ def get_members(obj: Dict[str, Any], cluster: Cluster, cluster_name: str, member
     if member_names:
         member_names = list(set(member_names) & candidates)
         if not member_names:
-            raise PatroniCtlException('No {0} among provided members'.format(role))
+            raise PatroniCtlException(
+                'No{0} among provided members'.format('t a single cluster member' if role == 'any' else ' ' + role))
     elif action != 'reinitialize':
         member_names = list(candidates)
 
@@ -945,43 +945,6 @@ def check_response(response: urllib3.response.HTTPResponse, member_name: str,
     return True
 
 
-def parse_scheduled(scheduled: Optional[str]) -> Optional[datetime.datetime]:
-    """Parse a string *scheduled* timestamp as a :class:`~datetime.datetime` object.
-
-    :param scheduled: string representation of the timestamp. May also be ``now``.
-
-    :returns: the corresponding :class:`~datetime.datetime` object, if *scheduled* is not ``now``, otherwise ``None``.
-
-    :raises:
-        :class:`PatroniCtlException`: if unable to parse *scheduled* from :class:`str` to :class:`~datetime.datetime`.
-
-    :Example:
-
-        >>> parse_scheduled(None) is None
-        True
-
-        >>> parse_scheduled('now') is None
-        True
-
-        >>> parse_scheduled('2023-05-29T04:32:31')
-        datetime.datetime(2023, 5, 29, 4, 32, 31, tzinfo=tzlocal())
-
-        >>> parse_scheduled('2023-05-29T04:32:31-3')
-        datetime.datetime(2023, 5, 29, 4, 32, 31, tzinfo=tzoffset(None, -10800))
-    """
-    if scheduled is not None and (scheduled or 'now') != 'now':
-        try:
-            scheduled_at = dateutil.parser.parse(scheduled)
-            if scheduled_at.tzinfo is None:
-                scheduled_at = scheduled_at.replace(tzinfo=dateutil.tz.tzlocal())
-        except (ValueError, TypeError):
-            message = 'Unable to parse scheduled timestamp ({0}). It should be in an unambiguous format (e.g. ISO 8601)'
-            raise PatroniCtlException(message.format(scheduled))
-        return scheduled_at
-
-    return None
-
-
 @ctl.command('reload', help='Reload cluster member configuration')
 @click.argument('cluster_name')
 @click.argument('member_names', nargs=-1)
@@ -1012,7 +975,6 @@ def reload(obj: Dict[str, Any], cluster_name: str, member_names: List[str],
         if r.status == 200:
             click.echo('No changes to apply on member {0}'.format(member.name))
         elif r.status == 202:
-            from patroni.config import get_global_config
             config = get_global_config(cluster)
             click.echo('Reload request received for member {0} and will be processed within {1} seconds'.format(
                 member.name, config.get('loop_wait') or dcs.loop_wait)
@@ -1062,16 +1024,20 @@ def restart(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member
             * *version* could not be parsed; or
             * a restart is attempted against a cluster that is in maintenance mode.
     """
+    action = 'restart'
     cluster = get_dcs(obj, cluster_name, group).get_cluster()
 
-    members = get_members(obj, cluster, cluster_name, member_names, role, force, 'restart', False, group=group)
+    members = get_members(obj, cluster, cluster_name, member_names, role, force, action, False, group=group)
     if scheduled is None and not force:
-        next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M')
+        next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M+00')
         scheduled = click.prompt('When should the restart take place (e.g. ' + next_hour + ') ',
                                  type=str, default='now')
+        scheduled = scheduled if scheduled != 'now' else None
 
-    scheduled_at = parse_scheduled(scheduled)
+    parse_result, scheduled_at = parse_schedule(scheduled)
-    confirm_members_action(members, force, 'restart', scheduled_at)
+    if parse_result:
+        raise PatroniCtlException(parse_result.value[0].format(action=action))
+    confirm_members_action(members, force, action, scheduled_at)
 
     if p_any:
         random.shuffle(members)
@@ -1094,7 +1060,6 @@ def restart(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member
         content['postgres_version'] = version
 
     if scheduled_at:
-        from patroni.config import get_global_config
         if get_global_config(cluster).is_paused:
             raise PatroniCtlException("Can't schedule restart in the paused state")
         content['schedule'] = scheduled_at.isoformat()
@@ -1214,6 +1179,9 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s
     click.echo('Current cluster topology')
     output_members(obj, cluster, cluster_name, group=group)
 
+    # Define everything missing via interactive input or available cluster info (if force mode)
+
+    # Require Citus group
     if obj.get('citus') and group is None:
         if force:
             raise PatroniCtlException('For Citus clusters the --group must me specified')
@@ -1222,72 +1190,82 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s
             dcs = get_dcs(obj, cluster_name, group)
             cluster = dcs.get_cluster()
 
-    if action == 'switchover' and (cluster.leader is None or not cluster.leader.name):
+    global_config = get_global_config(cluster)
-        raise PatroniCtlException('This cluster has no leader')
 
-    if leader is None:
+    # Leader is required for switchover only
-        if force or action == 'failover':
+    if action == 'switchover' and leader is None:
-            leader = cluster.leader and cluster.leader.name
+        if cluster.leader is None or not cluster.leader.name:
+            raise PatroniCtlException('This cluster has no leader')
+        if force:
+            leader = cluster.leader.name
         else:
-            from patroni.config import get_global_config
+            prompt = 'Standby Leader' if global_config.is_standby_cluster else 'Primary'
-            prompt = 'Standby Leader' if get_global_config(cluster).is_standby_cluster else 'Primary'
+            leader = click.prompt(prompt, type=str, default=(cluster.leader and cluster.leader.name))
-            leader = click.prompt(prompt, type=str, default=(cluster.leader and cluster.leader.member.name))
-
-    if leader is not None and cluster.leader and cluster.leader.member.name != leader:
-        raise PatroniCtlException('Member {0} is not the leader of cluster {1}'.format(leader, cluster_name))
-
-    # excluding members with nofailover tag
-    candidate_names = [str(m.name) for m in cluster.members if m.name != leader and not m.nofailover]
-    # We sort the names for consistent output to the client
-    candidate_names.sort()
-
-    if not candidate_names:
-        raise PatroniCtlException('No candidates found to {0} to'.format(action))
 
     if candidate is None and not force:
+        # Check if there are any candidates available at all
+        candidate_names = [str(m.name) for m in cluster.members if m.name != leader and not m.nofailover]
+        if not candidate_names:
+            raise PatroniCtlException('No candidates found to {0} to'.format(action))
+        candidate_names.sort()   # we sort the names for consistent output to the client
         candidate = click.prompt('Candidate ' + str(candidate_names), type=str, default='')
 
-    if action == 'failover' and not candidate:
+    # We allow manual failover to an aync node in the sync mode, so we better ask for the confirmation
-        raise PatroniCtlException('Failover could be performed only to a specific candidate')
+    if all((not force,
+            action == 'failover',
+            global_config.is_synchronous_mode,
+            not cluster.sync.is_empty,
+            not cluster.sync.matches(candidate, True))):
+        if click.confirm(f'Are you sure you want to failover to the asynchronous node {candidate}'):
+            raise PatroniCtlException('Aborting ' + action)
 
-    if candidate == leader:
+    if action == 'switchover' and scheduled is None and not force:
-        raise PatroniCtlException(action.title() + ' target and source are the same.')
+        next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M+00')
+        scheduled = click.prompt('When should the switchover take place (e.g. ' + next_hour + ') ',
+                                 type=str, default='now')
+        scheduled = scheduled if scheduled != 'now' else None
 
-    if candidate and candidate not in candidate_names:
+    # Now, when we collected all the possible info, run checks
-        raise PatroniCtlException('Member {0} does not exist in cluster {1}'.format(candidate, cluster_name))
+    manual_failover = ManualFailover(action, cluster, leader, candidate, scheduled,
+                                     global_config.is_paused, global_config.is_synchronous_mode)
+
+    result_text, _ = manual_failover.run_precheck().value
+    if result_text:
+        raise PatroniCtlException(result_text.format(action=action, leader=leader, candidate=candidate,
+                                                     cluster_name=cluster_name))
 
     scheduled_at_str = None
     scheduled_at = None
-
     if action == 'switchover':
-        if scheduled is None and not force:
+        parse_result, scheduled_at = manual_failover.parse_scheduled()
-            next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M')
+        if parse_result:
-            scheduled = click.prompt('When should the switchover take place (e.g. ' + next_hour + ' ) ',
+            raise PatroniCtlException(parse_result.value[0].format(action=action))
-                                     type=str, default='now')
-
-        scheduled_at = parse_scheduled(scheduled)
         if scheduled_at:
-            from patroni.config import get_global_config
-            if get_global_config(cluster).is_paused:
-                raise PatroniCtlException("Can't schedule switchover in the paused state")
             scheduled_at_str = scheduled_at.isoformat()
 
-    failover_value = {'leader': leader, 'candidate': candidate, 'scheduled_at': scheduled_at_str}
+    # By now we have established that the leader exists and the candidate exists,
-
+    # so confirm the action that is about to be run
-    logging.debug(failover_value)
-
-    # By now we have established that the leader exists and the candidate exists
     if not force:
-        demote_msg = ', demoting current leader ' + leader if leader else ''
+        demote_msg = f', demoting current leader {cluster.leader.name}' if cluster.leader else ''
         if scheduled_at_str:
-            if not click.confirm('Are you sure you want to schedule {0} of cluster {1} at {2}{3}?'
+            # only switchover can be scheduled
-                                 .format(action, cluster_name, scheduled_at_str, demote_msg)):
+            if not click.confirm(f'Are you sure you want to schedule a switchover in the cluster '
+                                 f'{cluster_name} at {scheduled_at_str}{demote_msg}?'):
+                # action as a var to catch a regression in the tests
                 raise PatroniCtlException('Aborting scheduled ' + action)
         else:
-            if not click.confirm('Are you sure you want to {0} cluster {1}{2}?'
+            if not click.confirm(f'Are you sure you want to perform a {action} in the cluster {cluster_name}{demote_msg}?'):
-                                 .format(action, cluster_name, demote_msg)):
                 raise PatroniCtlException('Aborting ' + action)
 
+    # And finally the actual work
+    failover_value = {'candidate': candidate}
+    if action == 'switchover':
+        failover_value['leader'] = leader
+    if scheduled_at_str:
+        failover_value['scheduled_at'] = scheduled_at_str
+
+    logging.debug(failover_value)
+
     r = None
     try:
         member = cluster.leader.member if cluster.leader else candidate and cluster.get_member(candidate, False)
@@ -1331,6 +1309,8 @@ def failover(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
 
     .. note::
         If *leader* is given perform a switchover instead of a failover.
+        This behavior is deprecated. ``--leader`` option support will be
+        removed in the next major release.
 
     .. seealso::
         Refer to :func:`_do_failover_or_switchover` for details.
@@ -1344,7 +1324,12 @@ def failover(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
     :param candidate: name of a standby member to be promoted. Nodes that are tagged with ``nofailover`` cannot be used.
     :param force: perform the failover or switchover without asking for confirmations.
     """
-    action = 'switchover' if leader else 'failover'
+    action = 'failover'
+    if leader:
+        action = 'switchover'
+        click.echo(click.style(
+            'Supplying a leader name using this command is deprecated and will be removed in a future version of'
+            ' Patroni, change your scripts to use `switchover` instead.\nExecuting switchover!', fg='red'))
     _do_failover_or_switchover(obj, action, cluster_name, group, leader, candidate, force)
 
 
@@ -1544,7 +1529,7 @@ def output_members(obj: Dict[str, Any], cluster: Cluster, name: str,
             logging.debug(member)
 
             lag = member.get('lag', '')
-            member.update(c=name, member=member['name'], group=g,
+            member.update(cluster=name, member=member['name'], group=g,
                           host=member.get('host', ''), tl=member.get('timeline', ''),
                           role=member['role'].replace('_', ' ').title(),
                           lag_in_mb=round(lag / 1024 / 1024) if isinstance(lag, int) else lag,
@@ -1562,9 +1547,11 @@ def output_members(obj: Dict[str, Any], cluster: Cluster, name: str,
             rows.append([member.get(n.lower().replace(' ', '_'), '') for n in columns])
 
     title = 'Citus cluster' if is_citus_cluster else 'Cluster'
-    group_title = '' if group is None else 'group: {0}, '.format(group)
+    title_details = f' ({initialize})'
-    title_details = group_title and ' ({0}{1})'.format(group_title, initialize)
+    if is_citus_cluster:
-    title = ' {0}: {1}{2} '.format(title, name, title_details)
+        title_details = '' if group is None else f' (group: {group}, {initialize})'
+
+    title = f' {title}: {name}{title_details} '
     print_output(columns, rows, {'Group': 'r', 'Lag in MB': 'r', 'TL': 'r'}, fmt, title)
 
     if fmt not in ('pretty', 'topology'):  # Omit service info when using machine-readable formats
@@ -1715,7 +1702,6 @@ def wait_until_pause_is_applied(dcs: AbstractDCS, paused: bool, old_cluster: Clu
     :param old_cluster: original cluster information before pause or unpause has been requested. Used to report which
         nodes are still pending to have ``pause`` equal *paused* at a given point in time.
     """
-    from patroni.config import get_global_config
     config = get_global_config(old_cluster)
 
     click.echo("'{0}' request sent, waiting until it is recognized by all nodes".format(paused and 'pause' or 'resume'))
@@ -1753,7 +1739,6 @@ def toggle_pause(config: Dict[str, Any], cluster_name: str, group: Optional[int]
             * ``pause`` state is already *paused*; or
             * cluster contains no accessible members.
     """
-    from patroni.config import get_global_config
     dcs = get_dcs(config, cluster_name, group)
     cluster = dcs.get_cluster()
     if get_global_config(cluster).is_paused == paused:
@@ -1817,7 +1802,7 @@ def resume(obj: Dict[str, Any], cluster_name: str, group: Optional[int], wait: b
 
 
 @contextmanager
-def temporary_file(contents: bytes, suffix: str = '', prefix: str = 'tmp') -> Generator[str, None, None]:
+def temporary_file(contents: bytes, suffix: str = '', prefix: str = 'tmp') -> Iterator[str]:
     """Create a temporary file with specified contents that persists for the context.
 
     :param contents: binary string that will be written to the file.

patroni/dcs/consul.py

@@ -15,7 +15,7 @@ from urllib3.exceptions import HTTPError
 from urllib.parse import urlencode, urlparse, quote
 from typing import Any, Callable, Dict, List, Mapping, NamedTuple, Optional, Union, Tuple, TYPE_CHECKING
 
-from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState,\
+from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState, \
     TimelineHistory, ReturnFalseException, catch_return_false_exception, citus_group_re
 from ..exceptions import DCSError
 from ..utils import deep_compare, parse_bool, Retry, RetryFailedError, split_host_port, uri, USER_AGENT
@@ -141,6 +141,36 @@ class HTTPClient(object):
 class ConsulClient(base.Consul):
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """
+        Consul client with Patroni customisations.
+
+        .. note::
+
+            Parameters, *token*, *cert* and *ca_cert* are not passed to the parent class :class:`consul.base.Consul`.
+
+        Original class documentation,
+
+            *token* is an optional ``ACL token``. If supplied it will be used by
+            default for all requests made with this client session. It's still
+            possible to override this token by passing a token explicitly for a
+            request.
+
+            *consistency* sets the consistency mode to use by default for all reads
+            that support the consistency option. It's still possible to override
+            this by passing explicitly for a given request. *consistency* can be
+            either 'default', 'consistent' or 'stale'.
+
+            *dc* is the datacenter that this agent will communicate with.
+            By default, the datacenter of the host is used.
+
+            *verify* is whether to verify the SSL certificate for HTTPS requests
+
+            *cert* client side certificates for HTTPS requests
+
+        :param args: positional arguments to pass to :class:`consul.base.Consul`
+        :param kwargs: keyword arguments, with *cert*, *ca_cert* and *token* removed, passed to
+                       :class:`consul.base.Consul`
+        """
         self._cert = kwargs.pop('cert', None)
         self._ca_cert = kwargs.pop('ca_cert', None)
         self.token = kwargs.get('token')
@@ -643,12 +673,8 @@ class Consul(AbstractDCS):
         return self._client.kv.put(self.history_path, value)
 
     @catch_consul_errors
-    def _delete_leader(self) -> bool:
+    def _delete_leader(self, leader: Leader) -> bool:
-        cluster = self.cluster
+        return self._client.kv.delete(self.leader_path, cas=int(leader.version))
-        if cluster and isinstance(cluster.leader, Leader) and\
-                cluster.leader.name == self._name and isinstance(cluster.leader.version, int):
-            return self._client.kv.delete(self.leader_path, cas=cluster.leader.version)
-        return True
 
     @catch_consul_errors
     def set_sync_state_value(self, value: str, version: Optional[int] = None) -> Union[int, bool]:

patroni/dcs/etcd.py

@@ -21,7 +21,7 @@ from urllib.parse import urlparse
 from urllib3 import Timeout
 from urllib3.exceptions import HTTPError, ReadTimeoutError, ProtocolError
 
-from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState,\
+from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState, \
     TimelineHistory, ReturnFalseException, catch_return_false_exception, citus_group_re
 from ..exceptions import DCSError
 from ..request import get as requests_get
@@ -809,7 +809,7 @@ class Etcd(AbstractEtcd):
         return bool(self.retry(self._client.write, self.initialize_path, sysid, prevExist=(not create_new)))
 
     @catch_etcd_errors
-    def _delete_leader(self) -> bool:
+    def _delete_leader(self, leader: Leader) -> bool:
         return bool(self._client.delete(self.leader_path, prevValue=self._name))
 
     @catch_etcd_errors

patroni/dcs/etcd3.py

@@ -15,7 +15,7 @@ from urllib3.exceptions import ReadTimeoutError, ProtocolError
 from threading import Condition, Lock, Thread
 from typing import Any, Callable, Collection, Dict, Iterator, List, Optional, Tuple, Type, TYPE_CHECKING, Union
 
-from . import ClusterConfig, Cluster, Failover, Leader, Member, SyncState,\
+from . import ClusterConfig, Cluster, Failover, Leader, Member, SyncState, \
     TimelineHistory, catch_return_false_exception, citus_group_re
 from .etcd import AbstractEtcdClientWithFailover, AbstractEtcd, catch_etcd_errors, DnsCachingResolver, Retry
 from ..exceptions import DCSError, PatroniException
@@ -228,7 +228,7 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
         return self.http.urlopen
 
     def _handle_server_response(self, response: urllib3.response.HTTPResponse) -> Dict[str, Any]:
-        data: Union[bytes, str] = response.data
+        data = response.data
         try:
             data = data.decode('utf-8')
             ret: Dict[str, Any] = json.loads(data)
@@ -630,6 +630,16 @@ class PatroniEtcd3Client(Etcd3Client):
 
         return ret
 
+    def txn(self, compare: Dict[str, Any], success: Dict[str, Any],
+            failure: Optional[Dict[str, Any]] = None, retry: Optional[Retry] = None) -> Dict[str, Any]:
+        ret = super(PatroniEtcd3Client, self).txn(compare, success, failure, retry)
+        # Here we abuse the fact that the `failure` is only set in the call from update_leader().
+        # In all other cases the txn() call failure may be an indicator of a stale cache,
+        # and therefore we want to restart watcher.
+        if not failure and not ret:
+            self._restart_watcher()
+        return ret
+
 
 class Etcd3(AbstractEtcd):
 
@@ -902,11 +912,10 @@ class Etcd3(AbstractEtcd):
         return self.retry(self._client.put, self.initialize_path, sysid, create_revision='0' if create_new else None)
 
     @catch_etcd_errors
-    def _delete_leader(self) -> bool:
+    def _delete_leader(self, leader: Leader) -> bool:
-        cluster = self.cluster
+        fields = build_range_request(self.leader_path)
-        if cluster and isinstance(cluster.leader, Leader) and cluster.leader.name == self._name:
+        compare = {'key': fields['key'], 'target': 'VALUE', 'value': base64_encode(self._name)}
-            return self._client.deleterange(self.leader_path, mod_revision=cluster.leader.version)
+        return bool(self._client.txn(compare, {'request_delete_range': fields}))
-        return True
 
     @catch_etcd_errors
     def cancel_initialization(self) -> bool:

patroni/dcs/kubernetes.py

@@ -19,10 +19,10 @@ from urllib3.exceptions import HTTPError
 from threading import Condition, Lock, Thread
 from typing import Any, Callable, Collection, Dict, List, Optional, Tuple, Type, Union, TYPE_CHECKING
 
-from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState,\
+from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState, \
     TimelineHistory, CITUS_COORDINATOR_GROUP_ID, citus_group_re
 from ..exceptions import DCSError
-from ..utils import deep_compare, iter_response_objects, keepalive_socket_options,\
+from ..utils import deep_compare, iter_response_objects, keepalive_socket_options, \
     Retry, RetryFailedError, tzutc, uri, USER_AGENT
 if TYPE_CHECKING:  # pragma: no cover
     from ..config import Config
@@ -134,6 +134,8 @@ class K8sConfig(object):
             config: Dict[str, Any] = yaml.safe_load(f)
 
         context = context or config['current-context']
+        if TYPE_CHECKING:  # pragma: no cover
+            assert isinstance(context, str)
         context_value = self._get_by_name(config, 'context', context)
         if TYPE_CHECKING:  # pragma: no cover
             assert isinstance(context_value, dict)
@@ -752,6 +754,10 @@ class Kubernetes(AbstractDCS):
         self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
         self._namespace = config.get('namespace') or 'default'
         self._role_label = config.get('role_label', 'role')
+        self._leader_label_value = config.get('leader_label_value', 'master')
+        self._follower_label_value = config.get('follower_label_value', 'replica')
+        self._standby_leader_label_value = config.get('standby_leader_label_value', 'master')
+        self._tmp_role_label = config.get('tmp_role_label')
         self._ca_certs = os.environ.get('PATRONI_KUBERNETES_CACERT', config.get('cacert')) or SERVICE_CERT_FILENAME
         super(Kubernetes, self).__init__({**config, 'namespace': ''})
         if self._citus_group:
@@ -1134,6 +1140,13 @@ class Kubernetes(AbstractDCS):
         """Unused"""
         raise NotImplementedError  # pragma: no cover
 
+    def write_leader_optime(self, last_lsn: int) -> None:
+        """Write value for WAL LSN to ``optime`` annotation of the leader object.
+
+        :param last_lsn: absolute WAL LSN in bytes.
+        """
+        self.patch_or_create(self.leader_path, {self._OPTIME: str(last_lsn)}, patch=True, retry=False)
+
     def _update_leader_with_retry(self, annotations: Dict[str, Any],
                                   resource_version: Optional[str], ips: List[str]) -> bool:
         retry = self._retry.copy()
@@ -1263,19 +1276,27 @@ class Kubernetes(AbstractDCS):
     def touch_member(self, data: Dict[str, Any]) -> bool:
         cluster = self.cluster
         if cluster and cluster.leader and cluster.leader.name == self._name:
-            role = 'master'
+            role = self._standby_leader_label_value if data['role'] == 'standby_leader' else self._leader_label_value
+            tmp_role = 'master'
         elif data['state'] == 'running' and data['role'] not in ('master', 'primary'):
-            role = data['role']
+            role = {'replica': self._follower_label_value}.get(data['role'], data['role'])
+            tmp_role = data['role']
         else:
             role = None
+            tmp_role = None
+
+        role_labels = {self._role_label: role}
+        if self._tmp_role_label:
+            role_labels[self._tmp_role_label] = tmp_role
 
         member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
         pod_labels = member and member.data.pop('pod_labels', None)
         ret = member and pod_labels is not None\
-            and pod_labels.get(self._role_label) == role and deep_compare(data, member.data)
+            and all(pod_labels.get(k) == v for k, v in role_labels.items())\
+            and deep_compare(data, member.data)
 
         if not ret:
-            metadata = {'namespace': self._namespace, 'name': self._name, 'labels': {self._role_label: role},
+            metadata = {'namespace': self._namespace, 'name': self._name, 'labels': role_labels,
                         'annotations': {'status': json.dumps(data, separators=(',', ':'))}}
             body = k8s_client.V1Pod(metadata=k8s_client.V1ObjectMeta(**metadata))
             ret = self._api.patch_namespaced_pod(self._name, self._namespace, body)
@@ -1291,11 +1312,11 @@ class Kubernetes(AbstractDCS):
             if cluster and cluster.config and cluster.config.version else None
         return self.patch_or_create_config({self._INITIALIZE: sysid}, resource_version)
 
-    def _delete_leader(self) -> bool:
+    def _delete_leader(self, leader: Leader) -> bool:
         """Unused"""
         raise NotImplementedError  # pragma: no cover
 
-    def delete_leader(self, last_lsn: Optional[int] = None) -> bool:
+    def delete_leader(self, leader: Optional[Leader], last_lsn: Optional[int] = None) -> bool:
         ret = False
         kind = self._kinds.get(self.leader_path)
         if kind and (kind.metadata.annotations or {}).get(self._LEADER) == self._name:

patroni/dcs/raft.py

@@ -446,7 +446,7 @@ class Raft(AbstractDCS):
     def initialize(self, create_new: bool = True, sysid: str = '') -> bool:
         return self._sync_obj.set(self.initialize_path, sysid, prevExist=(not create_new)) is not False
 
-    def _delete_leader(self) -> bool:
+    def _delete_leader(self, leader: Leader) -> bool:
         return self._sync_obj.delete(self.leader_path, prevValue=self._name, timeout=1)
 
     def cancel_initialization(self) -> bool:

patroni/dcs/zookeeper.py

@@ -466,7 +466,7 @@ class ZooKeeper(AbstractDCS):
                 return False
         return True
 
-    def _delete_leader(self) -> bool:
+    def _delete_leader(self, leader: Leader) -> bool:
         self._client.restart()
         return True
 

patroni/file_perm.py

@@ -0,0 +1,95 @@
+"""Helper object that helps with figuring out file and directory permissions based on permissions of PGDATA.
+
+:var logger: logger of this module.
+:var pg_perm: instance of the :class:`__FilePermissions` object.
+"""
+import logging
+import os
+import stat
+
+logger = logging.getLogger(__name__)
+
+
+class __FilePermissions:
+    """Helper class for managing permissions of directories and files under PGDATA.
+
+    Execute :meth:`set_permissions_from_data_directory` to figure out which permissions should be used for files and
+    directories under PGDATA based on permissions of PGDATA root directory.
+    """
+
+    # Mode mask for data directory permissions that only allows the owner to
+    # read/write directories and files -- mask 077.
+    __PG_MODE_MASK_OWNER = stat.S_IRWXG | stat.S_IRWXO
+
+    # Mode mask for data directory permissions that also allows group read/execute -- mask 027.
+    __PG_MODE_MASK_GROUP = stat.S_IWGRP | stat.S_IRWXO
+
+    # Default mode for creating directories -- mode 700.
+    __PG_DIR_MODE_OWNER = stat.S_IRWXU
+
+    # Mode for creating directories that allows group read/execute -- mode 750.
+    __PG_DIR_MODE_GROUP = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP
+
+    # Default mode for creating files -- mode 600.
+    __PG_FILE_MODE_OWNER = stat.S_IRUSR | stat.S_IWUSR
+
+    # Mode for creating files that allows group read -- mode 640.
+    __PG_FILE_MODE_GROUP = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP
+
+    def __init__(self) -> None:
+        """Create a :class:`__FilePermissions` object and set default permissions."""
+        self.__set_owner_permissions()
+        self.__set_umask()
+
+    def __set_umask(self) -> None:
+        """Set umask value based on calculations.
+
+        .. note::
+            Should only be called once either :meth:`__set_owner_permissions`
+            or :meth:`__set_group_permissions` has been executed.
+        """
+        try:
+            os.umask(self.__pg_mode_mask)
+        except Exception as e:
+            logger.error('Can not set umask to %03o: %r', self.__pg_mode_mask, e)
+
+    def __set_owner_permissions(self) -> None:
+        """Make directories/files accessible only by the owner."""
+        self.__pg_dir_create_mode = self.__PG_DIR_MODE_OWNER
+        self.__pg_file_create_mode = self.__PG_FILE_MODE_OWNER
+        self.__pg_mode_mask = self.__PG_MODE_MASK_OWNER
+
+    def __set_group_permissions(self) -> None:
+        """Make directories/files accessible by the owner and readable by group."""
+        self.__pg_dir_create_mode = self.__PG_DIR_MODE_GROUP
+        self.__pg_file_create_mode = self.__PG_FILE_MODE_GROUP
+        self.__pg_mode_mask = self.__PG_MODE_MASK_GROUP
+
+    def set_permissions_from_data_directory(self, data_dir: str) -> None:
+        """Set new permissions based on provided *data_dir*.
+
+        :param data_dir: reference to PGDATA to calculate permissions from.
+        """
+        try:
+            st = os.stat(data_dir)
+            if (st.st_mode & self.__PG_DIR_MODE_GROUP) == self.__PG_DIR_MODE_GROUP:
+                self.__set_group_permissions()
+            else:
+                self.__set_owner_permissions()
+        except Exception as e:
+            logger.error('Can not check permissions on %s: %r', data_dir, e)
+        else:
+            self.__set_umask()
+
+    @property
+    def dir_create_mode(self) -> int:
+        """Directory permissions."""
+        return self.__pg_dir_create_mode
+
+    @property
+    def file_create_mode(self) -> int:
+        """File permissions."""
+        return self.__pg_file_create_mode
+
+
+pg_perm = __FilePermissions()

patroni/ha.py

@@ -20,31 +20,28 @@ from .postgresql.callback_executor import CallbackAction
 from .postgresql.misc import postgres_version_to_int
 from .postgresql.postmaster import PostmasterProcess
 from .postgresql.rewind import Rewind
+from .tags import Tags
 from .utils import polling_loop, tzutc
 
 logger = logging.getLogger(__name__)
 
 
-class _MemberStatus(NamedTuple):
+class _MemberStatus(Tags, NamedTuple('_MemberStatus',
-    """Node status distilled from API response:
+                                     [('member', Member),
+                                      ('reachable', bool),
+                                      ('in_recovery', Optional[bool]),
+                                      ('wal_position', int),
+                                      ('data', Dict[str, Any])])):
+    """Node status distilled from API response.
 
-        member - dcs.Member object of the node
+    Consists of the following fields:
-        reachable - `!False` if the node is not reachable or is not responding with correct JSON
+
-        in_recovery - `!True` if pg_is_in_recovery() == true
+    :ivar member: :class:`~patroni.dcs.Member` object of the node.
-        dcs_last_seen - timestamp from JSON of last succesful communication with DCS
+    :ivar reachable: ``False`` if the node is not reachable or is not responding with correct JSON.
-        timeline - timeline value from JSON
+    :ivar in_recovery: ``False`` if the node is running as a primary (`if pg_is_in_recovery() == true`).
-        wal_position - maximum value of `replayed_location` or `received_location` from JSON
+    :ivar wal_position: maximum value of ``replayed_location`` or ``received_location`` from JSON.
-        tags - dictionary with values of different tags (i.e. nofailover)
+    :ivar data: the whole JSON response for future usage.
-        watchdog_failed - indicates that watchdog is required by configuration but not available or failed
     """
-    member: Member
-    reachable: bool
-    in_recovery: Optional[bool]
-    dcs_last_seen: int
-    timeline: int
-    wal_position: int
-    tags: Dict[str, Any]
-    watchdog_failed: bool
 
     @classmethod
     def from_api_response(cls, member: Member, json: Dict[str, Any]) -> '_MemberStatus':
@@ -56,22 +53,35 @@ class _MemberStatus(NamedTuple):
         # If one of those is not in a response we want to count the node as not healthy/reachable
         wal: Dict[str, Any] = json.get('wal') or json['xlog']
         # abuse difference in primary/replica response format
-        in_recovery = not bool(wal.get('location')) or json.get('role') in ('master', 'primary')
+        in_recovery = not (bool(wal.get('location')) or json.get('role') in ('master', 'primary'))
-        timeline = json.get('timeline', 0)
-        dcs_last_seen = json.get('dcs_last_seen', 0)
         lsn = int(in_recovery and max(wal.get('received_location', 0), wal.get('replayed_location', 0)))
-        return cls(member, True, in_recovery, dcs_last_seen, timeline, lsn,
+        return cls(member, True, in_recovery, lsn, json)
-                   json.get('tags', {}), json.get('watchdog_failed', False))
+
+    @property
+    def tags(self) -> Dict[str, Any]:
+        """Dictionary with values of different tags (i.e. nofailover)."""
+        return self.data.get('tags', {})
+
+    @property
+    def timeline(self) -> int:
+        """Timeline value from JSON."""
+        return self.data.get('timeline', 0)
+
+    @property
+    def watchdog_failed(self) -> bool:
+        """Indicates that watchdog is required by configuration but not available or failed."""
+        return self.data.get('watchdog_failed', False)
 
     @classmethod
     def unknown(cls, member: Member) -> '_MemberStatus':
-        return cls(member, False, None, 0, 0, 0, {}, False)
+        """Create a new class instance with empty or null values."""
+        return cls(member, False, None, 0, {})
 
     def failover_limitation(self) -> Optional[str]:
         """Returns reason why this node can't promote or None if everything is ok."""
         if not self.reachable:
             return 'not reachable'
-        if self.tags.get('nofailover', False):
+        if self.nofailover:
             return 'not allowed to promote'
         if self.watchdog_failed:
             return 'not watchdog capable'
@@ -83,11 +93,7 @@ class Failsafe(object):
     def __init__(self, dcs: AbstractDCS) -> None:
         self._lock = RLock()
         self._dcs = dcs
-        self._last_update = 0
+        self._reset_state()
-        self._name = None
-        self._conn_url = None
-        self._api_url = None
-        self._slots = None
 
     def update(self, data: Dict[str, Any]) -> None:
         with self._lock:
@@ -97,13 +103,20 @@ class Failsafe(object):
             self._api_url = data['api_url']
             self._slots = data.get('slots')
 
+    def _reset_state(self) -> None:
+        self._last_update = 0
+        self._name = None
+        self._conn_url = None
+        self._api_url = None
+        self._slots = None
+
     @property
     def leader(self) -> Optional[Leader]:
         with self._lock:
             if self._last_update + self._dcs.ttl > time.time() and self._name:
-                return Leader('', '', RemoteMember.from_name_and_data(self._name, {'api_url': self._api_url,
+                return Leader('', '', RemoteMember(self._name, {'api_url': self._api_url,
-                                                                                   'conn_url': self._conn_url,
+                                                                'conn_url': self._conn_url,
-                                                                                   'slots': self._slots}))
+                                                                'slots': self._slots}))
 
     def update_cluster(self, cluster: Cluster) -> Cluster:
         # Enreach cluster with the real leader if there was a ping from it
@@ -130,6 +143,8 @@ class Failsafe(object):
     def set_is_active(self, value: float) -> None:
         with self._lock:
             self._last_update = value
+            if not value:
+                self._reset_state()
 
 
 class Ha(object):
@@ -142,8 +157,8 @@ class Ha(object):
         self.cluster = Cluster.empty()
         self.global_config = self.patroni.config.get_global_config(None)
         self.old_cluster = Cluster.empty()
-        self._is_leader = False
+        self._leader_expiry = 0
-        self._is_leader_lock = RLock()
+        self._leader_expiry_lock = RLock()
         self._failsafe = Failsafe(patroni.dcs)
         self._was_paused = False
         self._leader_timeline = None
@@ -188,12 +203,38 @@ class Ha(object):
         return self.global_config.is_standby_cluster
 
     def is_leader(self) -> bool:
-        with self._is_leader_lock:
+        """:returns: `True` if the current node is the leader, based on expiration set when it last held the key."""
-            return self._is_leader > time.time()
+        with self._leader_expiry_lock:
+            return self._leader_expiry > time.time()
 
     def set_is_leader(self, value: bool) -> None:
-        with self._is_leader_lock:
+        """Update the current node's view of it's own leadership status.
-            self._is_leader = time.time() + self.dcs.ttl if value else 0
+
+        Will update the expiry timestamp to match the dcs ttl if setting leadership to true,
+        otherwise will set the expiry to the past to immediately invalidate.
+
+        :param value: is the current node the leader.
+        """
+        with self._leader_expiry_lock:
+            self._leader_expiry = time.time() + self.dcs.ttl if value else 0
+
+    def sync_mode_is_active(self) -> bool:
+        """Check whether synchronous replication is requested and already active.
+
+        :returns: ``True`` if the primary already put its name into the ``/sync`` in DCS.
+        """
+        return self.is_synchronous_mode() and not self.cluster.sync.is_empty
+
+    def _get_failover_action_name(self) -> str:
+        """Return the currently requested manual failover action name or the default ``failover``.
+
+        :returns: :class:`str` representing the manually requested action (``manual failover`` if no leader
+            is specified in the ``/failover`` in DCS, ``switchover`` otherwise) or ``failover`` if
+            ``/failover`` is empty.
+        """
+        if not self.cluster.failover:
+            return 'failover'
+        return 'switchover' if self.cluster.failover.is_switchover else 'manual failover'
 
     def load_cluster_from_dcs(self) -> None:
         cluster = self.dcs.get_cluster()
@@ -442,16 +483,23 @@ class Ha(object):
         """Handle the case when postgres isn't running.
 
         Depending on the state of Patroni, DCS cluster view, and pg_controldata the following could happen:
-        - if ``primary_start_timeout`` is 0 and this node owns the leader lock, the lock
+
-          will be voluntarily released if there are healthy replicas to take it over.
+          - if ``primary_start_timeout`` is 0 and this node owns the leader lock, the lock
-        - if postgres was running as a ``primary`` and this node owns the leader lock, postgres is started as primary.
+            will be voluntarily released if there are healthy replicas to take it over.
-        - crash recover in a single-user mode is executed in the following cases:
+
-          - postgres was running as ``primary`` wasn't ``shut down`` cleanly and there is no leader in DCS
+          - if postgres was running as a ``primary`` and this node owns the leader lock, postgres is started as primary.
-          - postgres was running as ``replica`` wasn't ``shut down in recovery`` (cleanly)
+
-            and we need to run ``pg_rewind`` to join back to the cluster.
+          - crash recover in a single-user mode is executed in the following cases:
-        - ``pg_rewind`` is executed if it is necessary, or optinally, the data directory could
+
-           be removed if it is allowed by configuration.
+            - postgres was running as ``primary`` wasn't ``shut down`` cleanly and there is no leader in DCS
-        - after ``crash recovery`` and/or ``pg_rewind`` are executed, postgres is started in recovery.
+
+            - postgres was running as ``replica`` wasn't ``shut down in recovery`` (cleanly)
+              and we need to run ``pg_rewind`` to join back to the cluster.
+
+          - ``pg_rewind`` is executed if it is necessary, or optinally, the data directory could
+             be removed if it is allowed by configuration.
+
+          - after ``crash recovery`` and/or ``pg_rewind`` are executed, postgres is started in recovery.
 
         :returns: action message, describing what was performed.
         """
@@ -460,7 +508,7 @@ class Ha(object):
             if timeout == 0:
                 # We are requested to prefer failing over to restarting primary. But see first if there
                 # is anyone to fail over to.
-                if self.is_failover_possible(self.cluster.members):
+                if self.is_failover_possible():
                     self.watchdog.disable()
                     logger.info("Primary crashed. Failing over.")
                     self.demote('immediate')
@@ -473,7 +521,8 @@ class Ha(object):
 
         # timeout > 0 indicates that we still have the leader lock, and it was just updated
         if timeout\
-                and data.get('Database cluster state') in ('in production', 'shutting down', 'shut down')\
+                and data.get('Database cluster state') in ('in production', 'in crash recovery',
+                                                           'shutting down', 'shut down')\
                 and self.state_handler.state == 'crashed'\
                 and self.state_handler.role in ('primary', 'master')\
                 and not self.state_handler.config.recovery_conf_exists():
@@ -496,6 +545,7 @@ class Ha(object):
 
         role = 'replica'
         if self.has_lock() and not self.is_standby_cluster():
+            self._rewind.reset_state()  # we want to later trigger CHECKPOINT after promote
             msg = "starting as readonly because i had the session lock"
             node_to_follow = None
         else:
@@ -525,10 +575,17 @@ class Ha(object):
         return msg
 
     def _get_node_to_follow(self, cluster: Cluster) -> Union[Leader, Member, None]:
-        # determine the node to follow. If replicatefrom tag is set,
+        """Determine the node to follow.
-        # try to follow the node mentioned there, otherwise, follow the leader.
+
-        if self.is_standby_cluster() and (self.cluster.is_unlocked() or self.has_lock(False)):
+        :param cluster: the currently known cluster state from DCS.
+
+        :returns: the node which we should be replicating from.
+        """
+        # The standby leader or when there is no standby leader we want to follow
+        # the remote member, except when there is no standby leader in pause.
+        if self.is_standby_cluster() and (self.has_lock(False) or self.cluster.is_unlocked() and not self.is_paused()):
             node_to_follow = self.get_remote_member()
+        # If replicatefrom tag is set, try to follow the node mentioned there, otherwise, follow the leader.
         elif self.patroni.replicatefrom and self.patroni.replicatefrom != self.state_handler.name:
             node_to_follow = cluster.get_member(self.patroni.replicatefrom)
         else:
@@ -551,7 +608,7 @@ class Ha(object):
         if refresh:
             self.load_cluster_from_dcs()
 
-        is_leader = self.state_handler.is_leader()
+        is_leader = self.state_handler.is_primary()
 
         node_to_follow = self._get_node_to_follow(self.cluster)
 
@@ -620,11 +677,20 @@ class Ha(object):
         promoting standbys that were guaranteed to be replicating synchronously.
         """
         if self.is_synchronous_mode():
-            current = CaseInsensitiveSet(self.cluster.sync.members)
+            sync = self.cluster.sync
+            if sync.is_empty:
+                # corner case: we need to explicitly enable synchronous mode by updating the
+                # ``/sync`` key with the current leader name and empty members. In opposite case
+                # it will never be automatically enabled if there are not eligible candidates.
+                sync = self.dcs.write_sync_state(self.state_handler.name, None, version=sync.version)
+                if not sync:
+                    return logger.warning("Updating sync state failed")
+                logger.info("Enabled synchronous replication")
+
+            current = CaseInsensitiveSet(sync.members)
             picked, allow_promote = self.state_handler.sync_handler.current_state(self.cluster)
 
             if picked != current:
-                sync = self.cluster.sync
                 # update synchronous standby list in dcs temporarily to point to common nodes in current and picked
                 sync_common = current & allow_promote
                 if sync_common != current:
@@ -730,7 +796,7 @@ class Ha(object):
         """
         if not self.is_paused():
             if not self.watchdog.is_running and not self.watchdog.activate():
-                if self.state_handler.is_leader():
+                if self.state_handler.is_primary():
                     self.demote('immediate')
                     return 'Demoting self because watchdog could not be activated'
                 else:
@@ -746,7 +812,7 @@ class Ha(object):
                     self._async_response.reset()
                     return 'Promotion cancelled because the pre-promote script failed'
 
-        if self.state_handler.is_leader():
+        if self.state_handler.is_primary():
             # Inform the state handler about its primary role.
             # It may be unaware of it if postgres is promoted manually.
             self.state_handler.set_role('master')
@@ -768,18 +834,17 @@ class Ha(object):
                 self.state_handler.sync_handler.set_synchronous_standby_names(
                     CaseInsensitiveSet('*') if self.global_config.is_synchronous_mode_strict else CaseInsensitiveSet())
             if self.state_handler.role not in ('master', 'promoted', 'primary'):
-                def on_success():
+                # reset failsafe state when promote
-                    self._rewind.reset_state()
+                self._failsafe.set_is_active(0)
-                    logger.info("cleared rewind state after becoming the leader")
 
                 def before_promote():
                     self.notify_citus_coordinator('before_promote')
 
                 with self._async_response:
                     self._async_response.reset()
+
                 self._async_executor.try_run_async('promote', self.state_handler.promote,
-                                                   args=(self.dcs.loop_wait, self._async_response,
+                                                   args=(self.dcs.loop_wait, self._async_response, before_promote))
-                                                         before_promote, on_success))
             return promote_message
 
     def fetch_node_status(self, member: Member) -> _MemberStatus:
@@ -797,6 +862,8 @@ class Ha(object):
         return _MemberStatus.unknown(member)
 
     def fetch_nodes_statuses(self, members: List[Member]) -> List[_MemberStatus]:
+        if not members:
+            return []
         pool = ThreadPool(len(members))
         results = pool.map(self.fetch_node_status, members)  # Run API calls on members in parallel
         pool.close()
@@ -834,7 +901,7 @@ class Ha(object):
             data['slots'] = self.state_handler.slots()
         except Exception:
             logger.exception('Exception when called state_handler.slots()')
-        members = [RemoteMember.from_name_and_data(name, {'api_url': url})
+        members = [RemoteMember(name, {'api_url': url})
                    for name, url in failsafe.items() if name != self.state_handler.name]
         if not members:  # A sinlge node cluster
             return True
@@ -849,11 +916,33 @@ class Ha(object):
         """Returns if instance with an wal should consider itself unhealthy to be promoted due to replication lag.
 
         :param wal_position: Current wal position.
+
         :returns True when node is lagging
         """
         lag = (self.cluster.last_lsn or 0) - wal_position
         return lag > self.global_config.maximum_lag_on_failover
 
+    def has_members_eligible_to_promote(self, members: List[Member], reference_lsn: int = 0,
+                                        fast_path: bool = False) -> bool:
+        ret = False
+        cluster_timeline = self.cluster.timeline
+
+        for st in self.fetch_nodes_statuses(members):
+            not_allowed_reason = st.failover_limitation()
+            if not_allowed_reason:
+                logger.info('Member %s is %s', st.member.name, not_allowed_reason)
+            elif fast_path:
+                return True
+            elif reference_lsn and st.wal_position < reference_lsn or \
+                    not reference_lsn and self.is_lagging(st.wal_position):
+                logger.info('Member %s exceeds maximum replication lag', st.member.name)
+            elif self.check_timeline() and (not st.timeline or st.timeline < cluster_timeline):
+                logger.info('Timeline %s of member %s is behind the cluster timeline %s',
+                            st.timeline, st.member.name, cluster_timeline)
+            else:
+                ret = True
+        return ret
+
     def _is_healthiest_node(self, members: Collection[Member], check_replication_lag: bool = True) -> bool:
         """This method tries to determine whether I am healthy enough to became a new leader candidate or not."""
 
@@ -875,71 +964,60 @@ class Ha(object):
         # Prepare list of nodes to run check against
         members = [m for m in members if m.name != self.state_handler.name and not m.nofailover and m.api_url]
 
-        if members:
+        for st in self.fetch_nodes_statuses(members):
-            for st in self.fetch_nodes_statuses(members):
+            if st.failover_limitation() is None:
-                if st.failover_limitation() is None:
+                if st.in_recovery is False:
-                    if st.in_recovery is False:
+                    logger.warning('Primary (%s) is still alive', st.member.name)
-                        logger.warning('Primary (%s) is still alive', st.member.name)
+                    return False
+                if my_wal_position < st.wal_position:
+                    logger.info('Wal position of %s is ahead of my wal position', st.member.name)
+                    # In synchronous mode the former leader might be still accessible and even be ahead of us.
+                    # We should not disqualify himself from the leader race in such a situation.
+                    if not self.sync_mode_is_active() or not self.cluster.sync.leader_matches(st.member.name):
                         return False
-                    if my_wal_position < st.wal_position:
+                    logger.info('Ignoring the former leader being ahead of us')
-                        logger.info('Wal position of %s is ahead of my wal position', st.member.name)
-                        # In synchronous mode the former leader might be still accessible and even be ahead of us.
-                        # We should not disqualify himself from the leader race in such a situation.
-                        if not self.is_synchronous_mode() or self.cluster.sync.is_empty\
-                                or not self.cluster.sync.leader_matches(st.member.name):
-                            return False
-                        logger.info('Ignoring the former leader being ahead of us')
         return True
 
-    def is_failover_possible(self, members: List[Member], check_synchronous: Optional[bool] = True,
+    def is_failover_possible(self, *, cluster_lsn: int = 0, exclude_failover_candidate: bool = False) -> bool:
-                             cluster_lsn: Optional[int] = 0) -> bool:
+        """Checks whether any of the cluster members is allowed to promote and is healthy enough for that.
-        """Checks whether one of the members from the list can possibly win the leader race.
 
-        :param members: list of members to check
+        :param cluster_lsn: to calculate replication lag and exclude member if it is lagging.
-        :param check_synchronous: consider only members that are known to be listed in /sync key when sync replication.
+        :param exclude_failover_candidate: if ``True``, exclude :attr:`failover.candidate` from the members
-        :param cluster_lsn: to calculate replication lag and exclude member if it is laggin
+                                           list against which the failover possibility checks are run.
-        :returns: `True` if there are members eligible to be the new leader
+        :returns: `True` if there are members eligible to become the new leader.
         """
-        ret = False
+        candidates = self.get_failover_candidates(exclude_failover_candidate)
-        cluster_timeline = self.cluster.timeline
+
-        members = [m for m in members if m.name != self.state_handler.name and not m.nofailover and m.api_url]
+        action = self._get_failover_action_name()
-        if check_synchronous and self.is_synchronous_mode() and not self.cluster.sync.is_empty:
+        if self.is_synchronous_mode() and self.cluster.failover and self.cluster.failover.candidate and not candidates:
-            members = [m for m in members if self.cluster.sync.matches(m.name)]
+            logger.warning('%s candidate=%s does not match with sync_standbys=%s',
-        if members:
+                           action.title(), self.cluster.failover.candidate, self.cluster.sync.sync_standby)
-            for st in self.fetch_nodes_statuses(members):
+        elif not candidates:
-                not_allowed_reason = st.failover_limitation()
+            logger.warning('%s: candidates list is empty', action)
-                if not_allowed_reason:
+
-                    logger.info('Member %s is %s', st.member.name, not_allowed_reason)
+        return self.has_members_eligible_to_promote(candidates, cluster_lsn)
-                elif cluster_lsn and st.wal_position < cluster_lsn or\
-                        not cluster_lsn and self.is_lagging(st.wal_position):
-                    logger.info('Member %s exceeds maximum replication lag', st.member.name)
-                elif self.check_timeline() and (not st.timeline or st.timeline < cluster_timeline):
-                    logger.info('Timeline %s of member %s is behind the cluster timeline %s',
-                                st.timeline, st.member.name, cluster_timeline)
-                else:
-                    ret = True
-        else:
-            logger.warning('manual failover: members list is empty')
-        return ret
 
     def manual_failover_process_no_leader(self) -> Optional[bool]:
         """Handles manual failover/switchover when the old leader already stepped down.
 
         :returns: - `True` if the current node is the best candidate to become the new leader
                   - `None` if the current node is running as a primary and requested candidate doesn't exist
-                  """
+        """
         failover = self.cluster.failover
         if TYPE_CHECKING:  # pragma: no cover
             assert failover is not None
-        if failover.candidate:  # manual failover to specific member
+
-            if failover.candidate == self.state_handler.name:  # manual failover to me
+        action = self._get_failover_action_name()
+
+        if failover.candidate:  # manual failover/switchover to specific member
+            if failover.candidate == self.state_handler.name:  # manual failover/switchover to me
                 return True
             elif self.is_paused():
                 # Remove failover key if the node to failover has terminated to avoid waiting for it indefinitely
                 # In order to avoid attempts to delete this key from all nodes only the primary is allowed to do it.
                 if not self.cluster.get_member(failover.candidate, fallback_to_leader=False)\
-                        and self.state_handler.is_leader():
+                        and self.state_handler.is_primary():
-                    logger.warning("manual failover: removing failover key because failover candidate is not running")
+                    logger.warning("%s: removing failover key because failover candidate is not running", action)
                     self.dcs.manual_failover('', '', version=failover.version)
                     return None
                 return False
@@ -955,22 +1033,21 @@ class Ha(object):
                 st = self.fetch_node_status(member)
                 not_allowed_reason = st.failover_limitation()
                 if not_allowed_reason is None:  # node is healthy
-                    logger.info('manual failover: to %s, i am %s', st.member.name, self.state_handler.name)
+                    logger.info('%s: to %s, i am %s', action, st.member.name, self.state_handler.name)
                     return False
-                # we wanted to failover to specific member but it is not healthy
+                # we wanted to failover/switchover to specific member but it is not healthy
-                logger.warning('manual failover: member %s is %s', st.member.name, not_allowed_reason)
+                logger.warning('%s: member %s is %s', action, st.member.name, not_allowed_reason)
 
-            # at this point we should consider all members as a candidates for failover
+            # at this point we should consider all members as a candidates for failover/switchover
             # i.e. we assume that failover.candidate is None
         elif self.is_paused():
             return False
 
-        # try to pick some other members to failover and check that they are healthy
+        # try to pick some other members for switchover and check that they are healthy
-        if failover.leader:
+        if failover.is_switchover:
             if self.state_handler.name == failover.leader:  # I was the leader
-                # exclude me and desired member which is unhealthy (failover.candidate can be None)
+                # exclude desired member which is unhealthy if it was specified
-                members = [m for m in self.cluster.members if m.name not in (failover.candidate, failover.leader)]
+                if self.is_failover_possible(exclude_failover_candidate=bool(failover.candidate)):
-                if self.is_failover_possible(members):  # check that there are healthy members
                     return False
                 else:  # I was the leader and it looks like currently I am the only healthy member
                     return True
@@ -985,6 +1062,7 @@ class Ha(object):
         """Performs a series of checks to determine that the current node is the best candidate.
 
         In case if manual failover/switchover is requested it calls :func:`manual_failover_process_no_leader` method.
+
         :returns: `True` if the current node is among the best candidates to become the new leader.
         """
         if time.time() - self._released_leader_key_timestamp < self.dcs.ttl:
@@ -997,10 +1075,23 @@ class Ha(object):
             if ret is not None:  # continue if we just deleted the stale failover key as a leader
                 return ret
 
-        if self.state_handler.is_leader():
+        if self.state_handler.is_primary():
-            # in pause leader is the healthiest only when no initialize or sysid matches with initialize!
+            if self.is_paused():
-            return not self.is_paused() or not self.cluster.initialize\
+                # in pause leader is the healthiest only when no initialize or sysid matches with initialize!
-                or self.state_handler.sysid == self.cluster.initialize
+                return not self.cluster.initialize or self.state_handler.sysid == self.cluster.initialize
+
+            # We want to protect from the following scenario:
+            # 1. node1 is stressed so much that heart-beat isn't running regularly and the leader lock expires.
+            # 2. node2 promotes, gets heavy load and the situation described in 1 repeats.
+            # 3. Patroni on node1 comes back, notices that Postgres is running as primary but there is
+            #    no leader key and "happily" acquires the leader lock.
+            # That is, node1 discarded promotion of node2. To avoid it we want to detect timeline change.
+            my_timeline = self.state_handler.get_primary_timeline()
+            if my_timeline < self.cluster.timeline:
+                logger.warning('My timeline %s is behind last known cluster timeline %s',
+                               my_timeline, self.cluster.timeline)
+                return False
+            return True
 
         if self.is_paused():
             return False
@@ -1010,8 +1101,8 @@ class Ha(object):
 
         if self.cluster.failover:
             # When doing a switchover in synchronous mode only synchronous nodes and former leader are allowed to race
-            if self.is_synchronous_mode() and self.cluster.failover.leader and \
+            if self.cluster.failover.is_switchover and self.sync_mode_is_active() \
-                    not self.cluster.sync.is_empty and not self.cluster.sync.matches(self.state_handler.name, True):
+                    and not self.cluster.sync.matches(self.state_handler.name, True):
                 return False
             return self.manual_failover_process_no_leader() or False
 
@@ -1028,12 +1119,11 @@ class Ha(object):
                 if failsafe_members and self.state_handler.name not in failsafe_members:
                     return False
                 # Race among not only existing cluster members, but also all known members from the failsafe config
-                all_known_members += [RemoteMember.from_name_and_data(name, {'api_url': url})
+                all_known_members += [RemoteMember(name, {'api_url': url}) for name, url in failsafe_members.items()]
-                                      for name, url in failsafe_members.items()]
         all_known_members += self.cluster.members
 
         # When in sync mode, only last known primary and sync standby are allowed to promote automatically.
-        if self.is_synchronous_mode() and not self.cluster.sync.is_empty:
+        if self.sync_mode_is_active():
             if not self.cluster.sync.matches(self.state_handler.name, True):
                 return False
             # pick between synchronous candidates so we minimize unnecessary failovers/demotions
@@ -1046,7 +1136,7 @@ class Ha(object):
 
     def _delete_leader(self, last_lsn: Optional[int] = None) -> None:
         self.set_is_leader(False)
-        self.dcs.delete_leader(last_lsn)
+        self.dcs.delete_leader(self.cluster.leader, last_lsn)
         self.dcs.reset_cluster()
 
     def release_leader_key_voluntarily(self, last_lsn: Optional[int] = None) -> None:
@@ -1057,13 +1147,15 @@ class Ha(object):
     def demote(self, mode: str) -> Optional[bool]:
         """Demote PostgreSQL running as primary.
 
-        :param mode: One of offline, graceful or immediate.
+        :param mode: One of offline, graceful, immediate or immediate-nolock.
-            offline is used when connection to DCS is not available.
+                     ``offline`` is used when connection to DCS is not available.
-            graceful is used when failing over to another node due to user request. May only be called running async.
+                     ``graceful`` is used when failing over to another node due to user request. May only be called
-            immediate is used when we determine that we are not suitable for primary and want to failover quickly
+                     running async.
-                without regard for data durability. May only be called synchronously.
+                     ``immediate`` is used when we determine that we are not suitable for primary and want to failover
-            immediate-nolock is used when find out that we have lost the lock to be primary. Need to bring down
+                     quickly without regard for data durability. May only be called synchronously.
-                PostgreSQL as quickly as possible without regard for data durability. May only be called synchronously.
+                     ``immediate-nolock`` is used when find out that we have lost the lock to be primary. Need to bring
+                     down PostgreSQL as quickly as possible without regard for data durability. May only be called
+                     synchronously.
         """
         mode_control = {
             'offline':          dict(stop='fast',      checkpoint=False, release=False, offline=True,  async_req=False),  # noqa: E241,E501
@@ -1084,7 +1176,7 @@ class Ha(object):
             # It could happen if Postgres is still archiving the backlog of WAL files.
             # If we know that there are replicas that received the shutdown checkpoint
             # location, we can remove the leader key and allow them to start leader race.
-            if self.is_failover_possible(self.cluster.members, cluster_lsn=checkpoint_location):
+            if self.is_failover_possible(cluster_lsn=checkpoint_location):
                 self.state_handler.set_role('demoted')
                 with self._async_executor:
                     self.release_leader_key_voluntarily(checkpoint_location)
@@ -1174,39 +1266,35 @@ class Ha(object):
 
         :returns: action message if demote was initiated, None if no action was taken"""
         failover = self.cluster.failover
-        if not failover or (self.is_paused() and not self.state_handler.is_leader()):
+        # if there is no failover key or
+        # I am holding the lock but am not primary = I am the standby leader,
+        # then do nothing
+        if not failover or (self.is_paused() and not self.state_handler.is_primary()):
             return
 
+        action = self._get_failover_action_name()
+        bare_action = action.replace('manual ', '')
+
+        # it is not the time for the scheduled switchover yet, do nothing
         if (failover.scheduled_at and not
-            self.should_run_scheduled_action("failover", failover.scheduled_at, lambda:
+            self.should_run_scheduled_action(bare_action, failover.scheduled_at, lambda:
                                              self.dcs.manual_failover('', '', version=failover.version))):
             return
 
         if not failover.leader or failover.leader == self.state_handler.name:
             if not failover.candidate or failover.candidate != self.state_handler.name:
                 if not failover.candidate and self.is_paused():
-                    logger.warning('Failover is possible only to a specific candidate in a paused state')
+                    logger.warning('%s is possible only to a specific candidate in a paused state', action.title())
+                elif self.is_failover_possible():
+                    ret = self._async_executor.try_run_async(f'{action}: demote', self.demote, ('graceful',))
+                    return ret or f'{action}: demoting myself'
                 else:
-                    if self.is_synchronous_mode():
+                    logger.warning('%s: no healthy members found, %s is not possible',
-                        if failover.candidate and not self.cluster.sync.matches(failover.candidate):
+                                   action, bare_action)
-                            logger.warning('Failover candidate=%s does not match with sync_standbys=%s',
-                                           failover.candidate, self.cluster.sync.sync_standby)
-                            members = []
-                        else:
-                            members = [m for m in self.cluster.members if self.cluster.sync.matches(m.name)]
-                    else:
-                        members = [m for m in self.cluster.members
-                                   if not failover.candidate or m.name == failover.candidate]
-                    if self.is_failover_possible(members, False):  # check that there are healthy members
-                        ret = self._async_executor.try_run_async('manual failover: demote', self.demote, ('graceful',))
-                        return ret or 'manual failover: demoting myself'
-                    else:
-                        logger.warning('manual failover: no healthy members found, failover is not possible')
             else:
-                logger.warning('manual failover: I am already the leader, no need to failover')
+                logger.warning('%s: I am already the leader, no need to %s', action, bare_action)
         else:
-            logger.warning('manual failover: leader name does not match: %s != %s',
+            logger.warning('%s: leader name does not match: %s != %s', action, failover.leader, self.state_handler.name)
-                           failover.leader, self.state_handler.name)
 
         logger.info('Cleaning up failover key')
         self.dcs.manual_failover('', '', version=failover.version)
@@ -1255,7 +1343,7 @@ class Ha(object):
 
     def process_healthy_cluster(self) -> str:
         if self.has_lock():
-            if self.is_paused() and not self.state_handler.is_leader():
+            if self.is_paused() and not self.state_handler.is_primary():
                 if self.cluster.failover and self.cluster.failover.candidate == self.state_handler.name:
                     return 'waiting to become primary after promote...'
 
@@ -1263,6 +1351,7 @@ class Ha(object):
                     self._delete_leader()
                     return 'removed leader lock because postgres is not running as primary'
 
+            # update lock to avoid split-brain
             if self.update_lock(True):
                 msg = self.process_manual_failover_from_leader()
                 if msg is not None:
@@ -1287,7 +1376,7 @@ class Ha(object):
             else:
                 # Either there is no connection to DCS or someone else acquired the lock
                 logger.error('failed to update leader lock')
-                if self.state_handler.is_leader():
+                if self.state_handler.is_primary():
                     if self.is_paused():
                         return 'continue to run as primary after failing to update leader lock in DCS'
                     self.demote('immediate-nolock')
@@ -1456,13 +1545,11 @@ class Ha(object):
         self._async_executor.run_async(self._do_reinitialize, args=(cluster, ))
 
     def handle_long_action_in_progress(self) -> str:
-        """
+        """Figure out what to do with the task AsyncExecutor is performing."""
-        Figure out what to do with the task AsyncExecutor is performing.
-        """
         if self.has_lock() and self.update_lock():
             if self._async_executor.scheduled_action == 'doing crash recovery in a single user mode':
                 time_left = self.global_config.primary_start_timeout - (time.time() - self._crash_recovery_started)
-                if time_left <= 0 and self.is_failover_possible(self.cluster.members):
+                if time_left <= 0 and self.is_failover_possible():
                     logger.info("Demoting self because crash recovery is taking too long")
                     self.state_handler.cancellable.cancel(True)
                     self.demote('immediate')
@@ -1526,7 +1613,7 @@ class Ha(object):
             self.cancel_initialization()
 
         if result is None:
-            if not self.state_handler.is_leader():
+            if not self.state_handler.is_primary():
                 return 'waiting for end of recovery after bootstrap'
 
             self.state_handler.set_role('master')
@@ -1553,8 +1640,7 @@ class Ha(object):
         return 'initialized a new cluster'
 
     def handle_starting_instance(self) -> Optional[str]:
-        """Starting up PostgreSQL may take a long time. In case we are the leader we may want to
+        """Starting up PostgreSQL may take a long time. In case we are the leader we may want to fail over to."""
-        fail over to."""
 
         # Check if we are in startup, when paused defer to main loop for manual failovers.
         if not self.state_handler.check_for_startup() or self.is_paused():
@@ -1574,7 +1660,7 @@ class Ha(object):
             time_left = timeout - self.state_handler.time_in_state()
 
             if time_left <= 0:
-                if self.is_failover_possible(self.cluster.members):
+                if self.is_failover_possible():
                     logger.info("Demoting self because primary startup is taking too long")
                     self.demote('immediate')
                     return 'stopped PostgreSQL because of startup timeout'
@@ -1594,7 +1680,8 @@ class Ha(object):
     def set_start_timeout(self, value: Optional[int]) -> None:
         """Sets timeout for starting as primary before eligible for failover.
 
-        Must be called when async_executor is busy or in the main thread."""
+        Must be called when async_executor is busy or in the main thread.
+        """
         self._start_timeout = value
 
     def _run_cycle(self) -> str:
@@ -1614,6 +1701,9 @@ class Ha(object):
             else:
                 if self._was_paused:
                     self.state_handler.schedule_sanity_checks_after_pause()
+                    # during pause people could manually do something with Postgres, therefore we want
+                    # to double check rewind conditions on replicas and maybe run CHECKPOINT on the primary
+                    self._rewind.reset_state()
                 self._was_paused = False
 
             if not self.cluster.has_member(self.state_handler.name):
@@ -1707,7 +1797,7 @@ class Ha(object):
                 elif self.cluster.is_unlocked() and not self.is_paused():
                     # "bootstrap", but data directory is not empty
                     if not self.state_handler.cb_called and self.state_handler.is_running() \
-                            and not self.state_handler.is_leader():
+                            and not self.state_handler.is_primary():
                         self._join_aborted = True
                         logger.error('No initialize key in DCS and PostgreSQL is running as replica, aborting start')
                         logger.error('Please first start Patroni on the node running as primary')
@@ -1750,7 +1840,7 @@ class Ha(object):
                 create_slots = self._sync_replication_slots(False)
 
                 if not self.state_handler.cb_called:
-                    if not is_promoting and not self.state_handler.is_leader():
+                    if not is_promoting and not self.state_handler.is_primary():
                         self._rewind.trigger_check_diverged_lsn()
                     self.state_handler.call_nowait(CallbackAction.ON_START)
 
@@ -1775,7 +1865,7 @@ class Ha(object):
 
     def _handle_dcs_error(self) -> str:
         if not self.is_paused() and self.state_handler.is_running():
-            if self.state_handler.is_leader():
+            if self.state_handler.is_primary():
                 if self.is_failsafe_mode() and self.check_failsafe_topology():
                     self.set_is_leader(True)
                     self._failsafe.set_is_active(time.time())
@@ -1796,7 +1886,9 @@ class Ha(object):
         """Handles replication slots.
 
         :param dcs_failed: bool, indicates that communication with DCS failed (get_cluster() or update_leader())
-        :returns: list[str], replication slots names that should be copied from the primary"""
+
+        :returns: list[str], replication slots names that should be copied from the primary
+        """
 
         slots: List[str] = []
 
@@ -1848,8 +1940,9 @@ class Ha(object):
                     # It could happen if Postgres is still archiving the backlog of WAL files.
                     # If we know that there are replicas that received the shutdown checkpoint
                     # location, we can remove the leader key and allow them to start leader race.
-                    if self.is_failover_possible(self.cluster.members, cluster_lsn=checkpoint_location):
+
-                        self.dcs.delete_leader(checkpoint_location)
+                    if self.is_failover_possible(cluster_lsn=checkpoint_location):
+                        self.dcs.delete_leader(self.cluster.leader, checkpoint_location)
                         status['deleted'] = True
                     else:
                         self.dcs.write_leader_optime(checkpoint_location)
@@ -1866,7 +1959,7 @@ class Ha(object):
             if not self.state_handler.is_running():
                 if self.is_leader() and not status['deleted']:
                     checkpoint_location = self.state_handler.latest_checkpoint_location()
-                    self.dcs.delete_leader(checkpoint_location)
+                    self.dcs.delete_leader(self.cluster.leader, checkpoint_location)
                 self.touch_member()
             else:
                 # XXX: what about when Patroni is started as the wrong user that has access to the watchdog device
@@ -1885,15 +1978,16 @@ class Ha(object):
         return self.dcs.watch(leader_version, timeout)
 
     def wakeup(self) -> None:
-        """Call of this method will trigger the next run of HA loop if there is
+        """Trigger the next run of HA loop if there is no "active" leader watch request in progress.
-        no "active" leader watch request in progress.
+
         This usually happens on the leader or if the node is running async action"""
         self.dcs.event.set()
 
     def get_remote_member(self, member: Union[Leader, Member, None] = None) -> RemoteMember:
-        """ In case of standby cluster this will tel us from which remote
+        """Get remote member node to stream from.
-            member to stream. Config can be both patroni config or
+
-            cluster.config.data
+        In case of standby cluster this will tell us from which remote member to stream. Config can be both patroni
+        config or cluster.config.data.
         """
         data: Dict[str, Any] = {}
         cluster_params = self.global_config.get_standby_cluster_config()
@@ -1907,4 +2001,33 @@ class Ha(object):
                 data['conn_kwargs'] = conn_kwargs
 
         name = member.name if member else 'remote_member:{}'.format(uuid.uuid1())
-        return RemoteMember.from_name_and_data(name, data)
+        return RemoteMember(name, data)
+
+    def get_failover_candidates(self, exclude_failover_candidate: bool) -> List[Member]:
+        """Return a list of candidates for either manual or automatic failover.
+
+        Exclude non-sync members when in synchronous mode, the current node (its checks are always performed earlier)
+        and the candidate if required. If failover candidate exclusion is not requested and a candidate is specified
+        in the /failover key, return the candidate only.
+        The result is further evaluated in the caller :func:`Ha.is_failover_possible` to check if any member is actually
+        healthy enough and is allowed to poromote.
+
+        :param exclude_failover_candidate: if ``True``, exclude :attr:`failover.candidate` from the candidates.
+
+        :returns: a list of :class:`Member` ojects or an empty list if there is no candidate available.
+        """
+        failover = self.cluster.failover
+        exclude = [self.state_handler.name] + ([failover.candidate] if failover and exclude_failover_candidate else [])
+
+        def is_eligible(node: Member) -> bool:
+            # in synchronous mode we allow failover (not switchover!) to async node
+            if self.sync_mode_is_active() and not self.cluster.sync.matches(node.name)\
+                    and not (failover and failover.is_failover):
+                return False
+            # Don't spend time on "nofailover" nodes checking.
+            # We also don't need nodes which we can't query with the api in the list.
+            return node.name not in exclude and \
+                not node.nofailover and bool(node.api_url) and \
+                (not failover or not failover.candidate or node.name == failover.candidate)
+
+        return list(filter(is_eligible, self.cluster.members))

patroni/log.py

@@ -21,17 +21,17 @@ _LOGGER = logging.getLogger(__name__)
 def debug_exception(self: logging.Logger, msg: object, *args: Any, **kwargs: Any) -> None:
     """Add full stack trace info to debug log messages and partial to others.
 
-    Handle :func:`exception` calls for *self*.
+    Handle :func:`~self.exception` calls for *self*.
 
     .. note::
         * If *self* log level is set to ``DEBUG``, then issue a ``DEBUG`` message with the complete stack trace;
         * If *self* log level is ``INFO`` or higher, then issue an ``ERROR`` message with only the last line of
             the stack trace.
 
-    :param self: logger for which :func:`exception` will be processed.
+    :param self: logger for which :func:`~self.exception` will be processed.
     :param msg: the message related to the exception to be logged.
-    :param args: positional arguments to be passed to :func:`self.debug` or :func:`loger_obj.error`.
+    :param args: positional arguments to be passed to :func:`~self.debug` or :func:`~self.error`.
-    :param kwargs: keyword arguments to be passed to :func:`self.debug` or :func:`loger_obj.error`.
+    :param kwargs: keyword arguments to be passed to :func:`~self.debug` or :func:`~self.error`.
     """
     kwargs.pop("exc_info", False)
     if self.isEnabledFor(logging.DEBUG):
@@ -44,16 +44,16 @@ def debug_exception(self: logging.Logger, msg: object, *args: Any, **kwargs: Any
 def error_exception(self: logging.Logger, msg: object, *args: Any, **kwargs: Any) -> None:
     """Add full stack trace info to error messages.
 
-    Handle :func:`exception` calls for *self*.
+    Handle :func:`~self.exception` calls for *self*.
 
     .. note::
         * By default issue an ``ERROR`` message with the complete stack trace. If you do not want to show the complete
-        stack trace, call with ``exc_info=False``.
+          stack trace, call with ``exc_info=False``.
 
-    :param self: logger for which :func:`exception` will be processed.
+    :param self: logger for which :func:`~self.exception` will be processed.
     :param msg: the message related to the exception to be logged.
-    :param args: positional arguments to be passed to :func:`loger_obj.error`.
+    :param args: positional arguments to be passed to :func:`~self.error`.
-    :param kwargs: keyword arguments to be passed to :func:`loger_obj.error`.
+    :param kwargs: keyword arguments to be passed to :func:`~self.error`.
     """
     exc_info = kwargs.pop("exc_info", True)
     self.error(msg, *args, exc_info=exc_info, **kwargs)
@@ -140,7 +140,7 @@ class ProxyHandler(logging.Handler):
     def emit(self, record: logging.LogRecord) -> None:
         """Emit each log record that is handled.
 
-        Will push the log record down to :func:`handle` method of the currently configured log handler.
+        Will push the log record down to :func:`~logging.Handler.handle` method of the currently configured log handler.
 
         :param record: the record that was emitted.
         """
@@ -203,7 +203,7 @@ class PatroniLogger(Thread):
         self._root_logger.addHandler(self._proxy_handler)
 
     def update_loggers(self) -> None:
-        """Configure loggers' log level as defined in ``log.loggers` section of Patroni configuration.
+        """Configure loggers' log level as defined in ``log.loggers`` section of Patroni configuration.
 
         .. note::
             It creates logger objects that are not defined yet in the log manager.
@@ -281,7 +281,8 @@ class PatroniLogger(Thread):
 
         .. note::
             It is used to remove different handlers that were configured previous to a reload in the configuration,
-            e.g. if we are switching from :class:`RotatingFileHandler` to class:`StreamHandler` and vice-versa.
+            e.g. if we are switching from :class:`~logging.handlers.RotatingFileHandler` to
+            class:`~logging.StreamHandler` and vice-versa.
         """
         while True:
             with self.log_handler_lock:

patroni/manual_failover.py

@@ -0,0 +1,93 @@
+from enum import Enum
+from typing import Optional, Tuple, TYPE_CHECKING
+
+if TYPE_CHECKING:  # pragma: no cover
+    import datetime
+
+    from .dcs import Cluster
+    from .ha import Patroni
+    from .utils import ParseScheduleErrors
+
+from .utils import parse_schedule
+
+
+class ManualFailoverPrecheckStatus(Enum):
+    FAILOVER_NO_CANDIDATE = ('Failover could be performed only to a specific candidate', 400)
+    SWITCHOVER_NO_LEADER = ('Switchover could be performed only from a specific leader', 400)
+    SCHEDULED_FAILOVER = ("Failover can't be scheduled", 400)
+    SCHEDULED_SWITCHOVER_PAUSE = ("Can't schedule switchover in the paused state", 400)
+    SWITCHOVER_PAUSE_NO_CANDIDATE = ('Switchover is possible only to a specific candidate in a paused state', 400)
+    SWITCHOVER_TO_LEADER = ('Switchover target and source are the same', 400)
+
+    CLUSTER_NO_LEADER = ('Cluster {cluster_name} has no leader', 412)
+    LEADER_NOT_MEMBER = ('Member {leader} is not the leader of cluster {cluster_name}', 412)
+    CANDIDATE_NOT_SYNC_STANDBY = ('candidate name does not match with sync_standby', 412)
+    NO_SYNC_CANDIDATE = ('{action} is not possible: can not find sync_standby', 412)
+    ONLY_LEADER = ('{action} is not possible: cluster does not have members except leader', 412)
+    CANDIDATE_NOT_MEMEBER = ('Member {candidate} does not exist in cluster {cluster_name} or is tagged as nofailover',
+                             412)
+    NO_GOOD_CANDIDATES = ('{action} is not possible: no good candidates have been found', 412)
+
+    CHECK_PASSED = ('', None)
+
+
+class ManualFailover(object):
+
+    def __init__(self, action: str, cluster: 'Cluster',
+                 leader: Optional[str], candidate: Optional[str], scheduled: Optional[str],
+                 paused: bool = False, sync_mode: bool = False, patroni_obj: Optional['Patroni'] = None) -> None:
+        self.action = action
+        self.cluster = cluster
+        self.leader = leader
+        self.candidate = candidate
+        self.scheduled = scheduled
+        self.paused = paused
+        self.sync_mode = sync_mode
+        self.patroni = patroni_obj
+
+    def parse_scheduled(self) -> Tuple[Optional['ParseScheduleErrors'], Optional['datetime.datetime']]:
+        return parse_schedule(self.scheduled)
+
+    def run_precheck(self) -> ManualFailoverPrecheckStatus:
+        if self.action == 'failover' and not self.candidate:
+            return ManualFailoverPrecheckStatus.FAILOVER_NO_CANDIDATE
+        elif self.action == 'switchover' and not self.leader:
+            return ManualFailoverPrecheckStatus.SWITCHOVER_NO_LEADER
+
+        if self.scheduled:
+            if self.action == 'failover':
+                return ManualFailoverPrecheckStatus.SCHEDULED_FAILOVER
+            elif self.paused:
+                return ManualFailoverPrecheckStatus.SCHEDULED_SWITCHOVER_PAUSE
+
+        if self.paused and not self.candidate:
+            return ManualFailoverPrecheckStatus.SWITCHOVER_PAUSE_NO_CANDIDATE
+
+        if self.leader == self.candidate:
+            return ManualFailoverPrecheckStatus.SWITCHOVER_TO_LEADER
+
+        if self.action == 'switchover':
+            if self.cluster.leader is None or not self.cluster.leader.name:
+                return ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER
+            if self.cluster.leader.name != self.leader:
+                return ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER
+
+        if self.candidate:
+            if self.action == 'switchover' and self.sync_mode and not self.cluster.sync.matches(self.candidate):
+                return ManualFailoverPrecheckStatus.CANDIDATE_NOT_SYNC_STANDBY
+            members = [m for m in self.cluster.members if m.name == self.candidate]
+            if not members:
+                return ManualFailoverPrecheckStatus.CANDIDATE_NOT_MEMEBER
+        elif self.sync_mode:
+            members = [m for m in self.cluster.members if self.cluster.sync.matches(m.name)]
+            if not members:
+                return ManualFailoverPrecheckStatus.NO_SYNC_CANDIDATE
+        else:
+            members = [m for m in self.cluster.members if not self.cluster.leader or m.name != self.cluster.leader.name and m.api_url]
+            if not members:
+                return ManualFailoverPrecheckStatus.ONLY_LEADER
+
+        if self.patroni and not self.patroni.ha.has_members_eligible_to_promote(members, fast_path=True):
+            return ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES
+
+        return ManualFailoverPrecheckStatus.CHECK_PASSED

patroni/postgresql/__init__.py

@@ -12,13 +12,13 @@ from datetime import datetime
 from dateutil import tz
 from psutil import TimeoutExpired
 from threading import current_thread, Lock
-from typing import Any, Callable, Dict, Generator, List, Optional, Union, Tuple, TYPE_CHECKING
+from typing import Any, Callable, Dict, Iterator, List, Optional, Union, Tuple, TYPE_CHECKING
 
 from .bootstrap import Bootstrap
 from .callback_executor import CallbackAction, CallbackExecutor
 from .cancellable import CancellableSubprocess
 from .config import ConfigHandler, mtime
-from .connection import Connection, get_connection_cursor
+from .connection import ConnectionPool, get_connection_cursor
 from .citus import CitusHandler
 from .misc import parse_history, parse_lsn, postgres_major_version_to_int
 from .postmaster import PostmasterProcess
@@ -57,8 +57,8 @@ class Postgresql(object):
     TL_LSN = ("CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0 "
               "ELSE ('x' || pg_catalog.substr(pg_catalog.pg_{0}file_name("
               "pg_catalog.pg_current_{0}_{1}()), 1, 8))::bit(32)::int END, "  # primary timeline
-              "CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0 "
+              "CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0 ELSE "
-              "ELSE pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}_{1}(), '0/0')::bigint END, "  # write_lsn
+              "pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}{2}_{1}(), '0/0')::bigint END, "  # wal(_flush)?_lsn
               "pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), '0/0')::bigint, "
               "pg_catalog.pg_{0}_{1}_diff(COALESCE(pg_catalog.pg_last_{0}_receive_{1}(), '0/0'), '0/0')::bigint, "
               "pg_catalog.pg_is_in_recovery() AND pg_catalog.pg_is_{0}_replay_paused()")
@@ -79,7 +79,8 @@ class Postgresql(object):
         self.set_state('stopped')
 
         self._pending_restart = False
-        self._connection = Connection()
+        self.connection_pool = ConnectionPool()
+        self._connection = self.connection_pool.get('heartbeat')
         self.citus_handler = CitusHandler(self, config.get('citus'))
         self.config = ConfigHandler(self, config)
         self.config.check_directories()
@@ -120,9 +121,9 @@ class Postgresql(object):
 
         if self.is_running():  # we are "joining" already running postgres
             self.set_state('running')
-            self.set_role('master' if self.is_leader() else 'replica')
+            self.set_role('master' if self.is_primary() else 'replica')
             # postpone writing postgresql.conf for 12+ because recovery parameters are not yet known
-            if self.major_version < 120000 or self.is_leader():
+            if self.major_version < 120000 or self.is_primary():
                 self.config.write_postgresql_conf()
             hba_saved = self.config.replace_pg_hba()
             ident_saved = self.config.replace_pg_ident()
@@ -159,6 +160,11 @@ class Postgresql(object):
     def wal_name(self) -> str:
         return 'wal' if self._major_version >= 100000 else 'xlog'
 
+    @property
+    def wal_flush(self) -> str:
+        """For PostgreSQL 9.6 onwards we want to use pg_current_wal_flush_lsn()/pg_current_xlog_flush_location()."""
+        return '_flush' if self._major_version >= 90600 else ''
+
     @property
     def lsn_name(self) -> str:
         return 'lsn' if self._major_version >= 100000 else 'location'
@@ -173,6 +179,7 @@ class Postgresql(object):
         """Returns the monitoring query with a fixed number of fields.
 
         The query text is constructed based on current state in DCS and PostgreSQL version:
+
         1. function names depend on version. wal/lsn for v10+ and xlog/location for pre v10.
         2. for primary we query timeline_id (extracted from pg_walfile_name()) and pg_current_wal_lsn()
         3. for replicas we query pg_last_wal_receive_lsn(), pg_last_wal_replay_lsn(), and  pg_is_wal_replay_paused()
@@ -182,7 +189,8 @@ class Postgresql(object):
         7. if sync replication is enabled we query pg_stat_replication and aggregate the result.
            In addition to that we get current values of synchronous_commit and synchronous_standby_names GUCs.
 
-        If some conditions are not satisfied we simply put static values instead. E.g., NULL, 0, '', and so on."""
+        If some conditions are not satisfied we simply put static values instead. E.g., NULL, 0, '', and so on.
+        """
 
         extra = ", " + (("pg_catalog.current_setting('synchronous_commit'), "
                          "pg_catalog.current_setting('synchronous_standby_names'), "
@@ -211,7 +219,7 @@ class Postgresql(object):
         else:
             extra = "0, NULL, NULL, NULL, NULL, NULL, NULL" + extra
 
-        return ("SELECT " + self.TL_LSN + ", {2}").format(self.wal_name, self.lsn_name, extra)
+        return ("SELECT " + self.TL_LSN + ", {3}").format(self.wal_name, self.lsn_name, self.wal_flush, extra)
 
     @property
     def available_gucs(self) -> CaseInsensitiveSet:
@@ -270,7 +278,7 @@ class Postgresql(object):
 
         :returns: 'ok' if PostgreSQL is up, 'reject' if starting up, 'no_resopnse' if not up."""
 
-        r = self.config.local_connect_kwargs
+        r = self.connection_pool.conn_kwargs
         cmd = [self.pgcommand('pg_isready'), '-p', r['port'], '-d', self._database]
 
         # Host is not set if we are connecting via default unix socket
@@ -321,40 +329,50 @@ class Postgresql(object):
     def connection(self) -> Union['connection3', 'Connection3[Any]']:
         return self._connection.get()
 
-    def set_connection_kwargs(self, kwargs: Dict[str, Any]) -> None:
+    def _query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]:
-        self._connection.set_conn_kwargs(kwargs.copy())
+        """Execute *sql* query with *params* and optionally return results.
-        self.citus_handler.set_conn_kwargs(kwargs.copy())
 
-    def _query(self, sql: str, *params: Any) -> Union['Cursor[Any]', 'cursor']:
+        :param sql: SQL statement to execute.
-        """We are always using the same cursor, therefore this method is not thread-safe!!!
+        :param params: parameters to pass.
-        You can call it from different threads only if you are holding explicit `AsyncExecutor` lock,
+
-        because the main thread is always holding this lock when running HA cycle."""
+        :returns: a query response as a list of tuples if there is any.
-        cursor = None
+        :raises:
+            :exc:`~psycopg.Error` if had issues while executing *sql*.
+
+            :exc:`~patroni.exceptions.PostgresConnectionException`: if had issues while connecting to the database.
+
+            :exc:`~patroni.utils.RetryFailedError`: if it was detected that connection/query failed due to PostgreSQL
+            restart.
+        """
         try:
-            cursor = self._connection.cursor()
+            return self._connection.query(sql, *params)
-            cursor.execute(sql.encode('utf-8'), params or None)
+        except PostgresConnectionException as exc:
-            return cursor
-        except psycopg.Error as e:
-            if cursor and cursor.connection.closed == 0:
-                # When connected via unix socket, psycopg2 can't recoginze 'connection lost'
-                # and leaves `_cursor_holder.connection.closed == 0`, but psycopg2.OperationalError
-                # is still raised (what is correct). It doesn't make sense to continiue with existing
-                # connection and we will close it, to avoid its reuse by the `cursor` method.
-                if isinstance(e, psycopg.OperationalError):
-                    self._connection.close()
-                else:
-                    raise e
             if self.state == 'restarting':
-                raise RetryFailedError('cluster is being restarted')
+                raise RetryFailedError('cluster is being restarted') from exc
-            raise PostgresConnectionException('connection problems')
+            raise
 
-    def query(self, sql: str, *args: Any, **kwargs: Any) -> Union['Cursor[Any]', 'cursor']:
+    def query(self, sql: str, *params: Any, retry: bool = True) -> List[Tuple[Any, ...]]:
-        if not kwargs.get('retry', True):
+        """Execute *sql* query with *params* and optionally return results.
-            return self._query(sql, *args)
+
+        :param sql: SQL statement to execute.
+        :param params: parameters to pass.
+        :param retry: whether the query should be retried upon failure or given up immediately.
+
+        :returns: a query response as a list of tuples if there is any.
+        :raises:
+            :exc:`~psycopg.Error` if had issues while executing *sql*.
+
+            :exc:`~patroni.exceptions.PostgresConnectionException`: if had issues while connecting to the database.
+
+            :exc:`~patroni.utils.RetryFailedError`: if it was detected that connection/query failed due to PostgreSQL
+            restart or if retry deadline was exceeded.
+        """
+        if not retry:
+            return self._query(sql, *params)
         try:
-            return self.retry(self._query, sql, *args)
+            return self.retry(self._query, sql, *params)
-        except RetryFailedError as e:
+        except RetryFailedError as exc:
-            raise PostgresConnectionException(str(e))
+            raise PostgresConnectionException(str(exc)) from exc
 
     def pg_control_exists(self) -> bool:
         return os.path.isfile(self._pg_control)
@@ -408,7 +426,18 @@ class Postgresql(object):
         :param global_config: last known :class:`GlobalConfig` object
         """
         self._cluster_info_state = {}
-        if cluster and cluster.config and cluster.config.modify_version:
+
+        if global_config:
+            self._global_config = global_config
+
+        if not self._global_config:
+            return
+
+        if self._global_config.is_standby_cluster:
+            # Standby cluster can't have logical replication slots, and we don't need to enforce hot_standby_feedback
+            self._has_permanent_logical_slots = False
+            self.set_enforce_hot_standby_feedback(False)
+        elif cluster and cluster.config and cluster.config.modify_version:
             self._has_permanent_logical_slots =\
                 cluster.has_permanent_logical_slots(self.name, nofailover, self.major_version)
 
@@ -418,13 +447,10 @@ class Postgresql(object):
                 self._has_permanent_logical_slots
                 or cluster.should_enforce_hot_standby_feedback(self.name, nofailover, self.major_version))
 
-        if global_config:
-            self._global_config = global_config
-
     def _cluster_info_state_get(self, name: str) -> Optional[Any]:
         if not self._cluster_info_state:
             try:
-                result = self._is_leader_retry(self._query, self.cluster_info_query).fetchone()
+                result = self._is_leader_retry(self._query, self.cluster_info_query)[0]
                 cluster_info_state = dict(zip(['timeline', 'wal_position', 'replayed_location',
                                                'received_location', 'replay_paused', 'pg_control_timeline',
                                                'received_tli', 'slot_name', 'conninfo', 'receiver_state',
@@ -474,14 +500,14 @@ class Postgresql(object):
         """:returns: a result set of 'SELECT * FROM pg_stat_replication'."""
         return self._cluster_info_state_get('pg_stat_replication') or []
 
-    def replication_state_from_parameters(self, is_leader: bool, receiver_state: Optional[str],
+    def replication_state_from_parameters(self, is_primary: bool, receiver_state: Optional[str],
                                           restore_command: Optional[str]) -> Optional[str]:
         """Figure out the replication state from input parameters.
 
         .. note::
             This method could be only called when Postgres is up, running and queries are successfuly executed.
 
-        :is_leader: `True` is postgres is not running in recovery
+        :is_primary: `True` is postgres is not running in recovery
         :receiver_state: value from `pg_stat_get_wal_receiver.state` or None if Postgres is older than 9.6
         :restore_command: value of ``restore_command`` GUC for PostgreSQL 12+ or
                           `postgresql.recovery_conf.restore_command` if it is set in Patroni configuration
@@ -490,7 +516,7 @@ class Postgresql(object):
                   - 'streaming' if replica is streaming according to the `pg_stat_wal_receiver` view;
                   - 'in archive recovery' if replica isn't streaming and there is a `restore_command`
         """
-        if self._major_version >= 90600 and not is_leader:
+        if self._major_version >= 90600 and not is_primary:
             if receiver_state == 'streaming':
                 return 'streaming'
             # For Postgres older than 12 we get `restore_command` from Patroni config, otherwise we check GUC
@@ -505,11 +531,11 @@ class Postgresql(object):
 
         :returns: ``streaming``, ``in archive recovery``, or ``None``
         """
-        return self.replication_state_from_parameters(self.is_leader(),
+        return self.replication_state_from_parameters(self.is_primary(),
                                                       self._cluster_info_state_get('receiver_state'),
                                                       self._cluster_info_state_get('restore_command'))
 
-    def is_leader(self) -> bool:
+    def is_primary(self) -> bool:
         try:
             return bool(self._cluster_info_state_get('timeline'))
         except PostgresConnectionException:
@@ -665,7 +691,7 @@ class Postgresql(object):
         # the former node, otherwise, we might get a stalled one
         # after kill -9, which would report incorrect data to
         # patroni.
-        self._connection.close()
+        self.connection_pool.close()
 
         if self.is_running():
             logger.error('Cannot start PostgreSQL because one is already running.')
@@ -736,7 +762,7 @@ class Postgresql(object):
     def checkpoint(self, connect_kwargs: Optional[Dict[str, Any]] = None,
                    timeout: Optional[float] = None) -> Optional[str]:
         check_not_is_in_recovery = connect_kwargs is not None
-        connect_kwargs = connect_kwargs or self.config.local_connect_kwargs
+        connect_kwargs = connect_kwargs or self.connection_pool.conn_kwargs
         for p in ['connect_timeout', 'options']:
             connect_kwargs.pop(p, None)
         if timeout:
@@ -866,11 +892,10 @@ class Postgresql(object):
 
     def _wait_for_connection_close(self, postmaster: PostmasterProcess) -> None:
         try:
-            with self.connection().cursor() as cur:
+            while postmaster.is_running():  # Need a timeout here?
-                while postmaster.is_running():  # Need a timeout here?
+                self._connection.query("SELECT 1")
-                    cur.execute("SELECT 1")
+                time.sleep(STOP_POLLING_INTERVAL)
-                    time.sleep(STOP_POLLING_INTERVAL)
+        except (psycopg.Error, PostgresConnectionException):
-        except psycopg.Error:
             pass
 
     def reload(self, block_callbacks: bool = False) -> bool:
@@ -999,7 +1024,7 @@ class Postgresql(object):
 
     @contextmanager
     def get_replication_connection_cursor(self, host: Optional[str] = None, port: int = 5432,
-                                          **kwargs: Any) -> Generator[Union['cursor', 'Cursor[Any]'], None, None]:
+                                          **kwargs: Any) -> Iterator[Union['cursor', 'Cursor[Any]']]:
         conn_kwargs = self.config.replication.copy()
         conn_kwargs.update(host=host, port=int(port) if port else None, user=conn_kwargs.pop('username'),
                            connect_timeout=3, replication=1, options='-c statement_timeout=2000')
@@ -1125,8 +1150,8 @@ class Postgresql(object):
         except Exception as e:
             logger.error('Exception when calling `%s`: %r', cmd, e)
 
-    def promote(self, wait_seconds: int, task: CriticalTask, before_promote: Optional[Callable[..., Any]] = None,
+    def promote(self, wait_seconds: int, task: CriticalTask,
-                on_success: Optional[Callable[..., Any]] = None) -> Optional[bool]:
+                before_promote: Optional[Callable[..., Any]] = None) -> Optional[bool]:
         if self.role in ('promoted', 'master', 'primary'):
             return True
 
@@ -1152,16 +1177,14 @@ class Postgresql(object):
         ret = self.pg_ctl('promote', '-W')
         if ret:
             self.set_role('promoted')
-            if on_success is not None:
-                on_success()
             self.call_nowait(CallbackAction.ON_ROLE_CHANGE)
             ret = self._wait_promote(wait_seconds)
         return ret
 
     @staticmethod
-    def _wal_position(is_leader: bool, wal_position: int,
+    def _wal_position(is_primary: bool, wal_position: int,
                       received_location: Optional[int], replayed_location: Optional[int]) -> int:
-        return wal_position if is_leader else max(received_location or 0, replayed_location or 0)
+        return wal_position if is_primary else max(received_location or 0, replayed_location or 0)
 
     def timeline_wal_position(self) -> Tuple[int, int, Optional[int]]:
         # This method could be called from different threads (simultaneously with some other `_query` calls).
@@ -1173,31 +1196,21 @@ class Postgresql(object):
             received_location = self.received_location()
             pg_control_timeline = self._cluster_info_state_get('pg_control_timeline')
         else:
-            with self.connection().cursor() as cursor:
+            timeline, wal_position, replayed_location, received_location, _, pg_control_timeline = \
-                cursor.execute(self.cluster_info_query.encode('utf-8'))
+                self._query(self.cluster_info_query)[0][:6]
-                row = cursor.fetchone()
-                if TYPE_CHECKING:  # pragma: no cover
-                    assert row is not None
-                (timeline, wal_position, replayed_location, received_location, _, pg_control_timeline) = row[:6]
 
         wal_position = self._wal_position(bool(timeline), wal_position, received_location, replayed_location)
-        return (timeline, wal_position, pg_control_timeline)
+        return timeline, wal_position, pg_control_timeline
 
     def postmaster_start_time(self) -> Optional[str]:
         try:
-            query = "SELECT " + self.POSTMASTER_START_TIME
+            sql = "SELECT " + self.POSTMASTER_START_TIME
-            if current_thread().ident == self.__thread_ident:
+            return self.query(sql, retry=current_thread().ident == self.__thread_ident)[0][0].isoformat(sep=' ')
-                row = self.query(query).fetchone()
-            else:
-                with self.connection().cursor() as cursor:
-                    cursor.execute(query)
-                    row = cursor.fetchone()
-            return row[0].isoformat(sep=' ') if row else None
         except psycopg.Error:
             return None
 
     def last_operation(self) -> int:
-        return self._wal_position(self.is_leader(), self._cluster_info_state_get('wal_position') or 0,
+        return self._wal_position(self.is_primary(), self._cluster_info_state_get('wal_position') or 0,
                                   self.received_location(), self.replayed_location())
 
     def configure_server_parameters(self) -> None:

patroni/postgresql/bootstrap.py

@@ -176,7 +176,7 @@ class Bootstrap(object):
         """
         cmd = config.get('post_bootstrap') or config.get('post_init')
         if cmd:
-            r = self._postgresql.config.local_connect_kwargs
+            r = self._postgresql.connection_pool.conn_kwargs
             connstring = self._postgresql.config.format_dsn(r, True)
             if 'host' not in r:
                 # https://www.postgresql.org/docs/current/static/libpq-pgpass.html

patroni/postgresql/citus.py

@@ -6,13 +6,10 @@ from threading import Condition, Event, Thread
 from urllib.parse import urlparse
 from typing import Any, Dict, List, Optional, Union, Tuple, TYPE_CHECKING
 
-from .connection import Connection
 from ..dcs import CITUS_COORDINATOR_GROUP_ID, Cluster
 from ..psycopg import connect, quote_ident
 
 if TYPE_CHECKING:  # pragma: no cover
-    from psycopg import Cursor
-    from psycopg2 import cursor
     from . import Postgresql
 
 CITUS_SLOT_NAME_RE = re.compile(r'^citus_shard_(move|split)_slot(_[1-9][0-9]*){2,3}$')
@@ -73,7 +70,10 @@ class CitusHandler(Thread):
         self.daemon = True
         self._postgresql = postgresql
         self._config = config
-        self._connection = Connection()
+        if config:
+            self._connection = postgresql.connection_pool.get(
+                'citus', {'dbname': config['database'],
+                          'options': '-c statement_timeout=0 -c idle_in_transaction_session_timeout=0'})
         self._pg_dist_node: Dict[int, PgDistNode] = {}  # Cache of pg_dist_node: {groupid: PgDistNode()}
         self._tasks: List[PgDistNode] = []  # Requests to change pg_dist_node, every task is a `PgDistNode`
         self._in_flight: Optional[PgDistNode] = None  # Reference to the `PgDistNode` being changed in a transaction
@@ -93,12 +93,6 @@ class CitusHandler(Thread):
     def is_worker(self) -> bool:
         return self.is_enabled() and not self.is_coordinator()
 
-    def set_conn_kwargs(self, kwargs: Dict[str, Any]) -> None:
-        if isinstance(self._config, dict):  # self.is_enabled():
-            kwargs.update({'dbname': self._config['database'],
-                           'options': '-c statement_timeout=0 -c idle_in_transaction_session_timeout=0'})
-            self._connection.set_conn_kwargs(kwargs)
-
     def schedule_cache_rebuild(self) -> None:
         with self._condition:
             self._schedule_load_pg_dist_node = True
@@ -109,12 +103,10 @@ class CitusHandler(Thread):
             self._tasks[:] = []
             self._in_flight = None
 
-    def query(self, sql: str, *params: Any) -> Union['Cursor[Any]', 'cursor']:
+    def query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]:
         try:
             logger.debug('query(%s, %s)', sql, params)
-            cursor = self._connection.cursor()
+            return self._connection.query(sql, *params)
-            cursor.execute(sql.encode('utf-8'), params or None)
-            return cursor
         except Exception as e:
             logger.error('Exception when executing query "%s", (%s): %r', sql, params, e)
             self._connection.close()
@@ -132,13 +124,13 @@ class CitusHandler(Thread):
             self._schedule_load_pg_dist_node = False
 
         try:
-            cursor = self.query("SELECT nodeid, groupid, nodename, nodeport, noderole"
+            rows = self.query("SELECT nodeid, groupid, nodename, nodeport, noderole"
-                                " FROM pg_catalog.pg_dist_node WHERE noderole = 'primary'")
+                              " FROM pg_catalog.pg_dist_node WHERE noderole = 'primary'")
         except Exception:
             return False
 
         with self._condition:
-            self._pg_dist_node = {r[1]: PgDistNode(r[1], r[2], r[3], 'after_promote', r[0]) for r in cursor}
+            self._pg_dist_node = {r[1]: PgDistNode(r[1], r[2], r[3], 'after_promote', r[0]) for r in rows}
         return True
 
     def sync_pg_dist_node(self, cluster: Cluster) -> None:
@@ -174,11 +166,13 @@ class CitusHandler(Thread):
         """Returns the tuple(i, task), where `i` - is the task index in the self._tasks list
 
         Tasks are picked by following priorities:
+
         1. If there is already a transaction in progress, pick a task
            that that will change already affected worker primary.
         2. If the coordinator address should be changed - pick a task
            with group=0 (coordinators are always in group 0).
-        3. Pick a task that is the oldest (first from the self._tasks)"""
+        3. Pick a task that is the oldest (first from the self._tasks)
+        """
 
         with self._condition:
             if self._in_flight:
@@ -209,10 +203,8 @@ class CitusHandler(Thread):
             self.query('SELECT pg_catalog.citus_update_node(%s, %s, %s, true, %s)',
                        task.nodeid, task.host, task.port, task.cooldown)
         elif task.event != 'before_demote':
-            row = self.query("SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default')",
+            task.nodeid = self.query("SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default')",
-                             task.host, task.port, task.group).fetchone()
+                                     task.host, task.port, task.group)[0][0]
-            if row is not None:
-                task.nodeid = row[0]
 
     def process_task(self, task: PgDistNode) -> bool:
         """Updates a single row in `pg_dist_node` table, optionally in a transaction.
@@ -363,8 +355,8 @@ class CitusHandler(Thread):
         if not isinstance(self._config, dict):  # self.is_enabled()
             return
 
-        conn_kwargs = self._postgresql.config.local_connect_kwargs
+        conn_kwargs = {**self._postgresql.connection_pool.conn_kwargs,
-        conn_kwargs['options'] = '-c synchronous_commit=local -c statement_timeout=0'
+                       'options': '-c synchronous_commit=local -c statement_timeout=0'}
         if self._config['database'] != self._postgresql.database:
             conn = connect(**conn_kwargs)
             try:
@@ -412,7 +404,7 @@ class CitusHandler(Thread):
         parameters['wal_level'] = 'logical'
 
     def ignore_replication_slot(self, slot: Dict[str, str]) -> bool:
-        if isinstance(self._config, dict) and self._postgresql.is_leader() and\
+        if isinstance(self._config, dict) and self._postgresql.is_primary() and\
                 slot['type'] == 'logical' and slot['database'] == self._config['database']:
             m = CITUS_SLOT_NAME_RE.match(slot['name'])
             return bool(m and {'move': 'pgoutput', 'split': 'citus'}.get(m.group(1)) == slot['plugin'])

patroni/postgresql/config.py

@@ -6,16 +6,18 @@ import socket
 import stat
 import time
 
+from contextlib import contextmanager
 from urllib.parse import urlparse, parse_qsl, unquote
 from types import TracebackType
-from typing import Any, Collection, Dict, List, Optional, Union, Tuple, Type, TYPE_CHECKING
+from typing import Any, Collection, Dict, Iterator, List, Optional, Union, Tuple, Type, TYPE_CHECKING
 
 from .validator import recovery_parameters, transform_postgresql_parameter_value, transform_recovery_parameter_value
 from ..collections import CaseInsensitiveDict, CaseInsensitiveSet
 from ..dcs import Leader, Member, RemoteMember, slot_name_from_member_name
-from ..exceptions import PatroniFatalException
+from ..exceptions import PatroniFatalException, PostgresConnectionException
+from ..file_perm import pg_perm
 from ..utils import compare_values, parse_bool, parse_int, split_host_port, uri, validate_directory, is_subpath
-from ..validator import IntValidator
+from ..validator import IntValidator, EnumValidator
 
 if TYPE_CHECKING:  # pragma: no cover
     from . import Postgresql
@@ -258,14 +260,14 @@ def _false_validator(value: Any) -> bool:
     return False
 
 
-def _wal_level_validator(value: Any) -> bool:
-    return str(value).lower() in ('hot_standby', 'replica', 'logical')
-
-
 def _bool_validator(value: Any) -> bool:
     return parse_bool(value) is not None
 
 
+def _bool_is_true_validator(value: Any) -> bool:
+    return parse_bool(value) is True
+
+
 class ConfigHandler(object):
 
     # List of parameters which must be always passed to postmaster as command line options
@@ -286,8 +288,8 @@ class ConfigHandler(object):
         'listen_addresses': (None, _false_validator, 90100),
         'port': (None, _false_validator, 90100),
         'cluster_name': (None, _false_validator, 90500),
-        'wal_level': ('hot_standby', _wal_level_validator, 90100),
+        'wal_level': ('hot_standby', EnumValidator(('hot_standby', 'replica', 'logical')), 90100),
-        'hot_standby': ('on', _false_validator, 90100),
+        'hot_standby': ('on', _bool_is_true_validator, 90100),
         'max_connections': (100, IntValidator(min=25), 90100),
         'max_wal_senders': (10, IntValidator(min=3), 90100),
         'wal_keep_segments': (8, IntValidator(min=1), 90100),
@@ -297,7 +299,7 @@ class ConfigHandler(object):
         'track_commit_timestamp': ('off', _bool_validator, 90500),
         'max_replication_slots': (10, IntValidator(min=4), 90400),
         'max_worker_processes': (8, IntValidator(min=2), 90400),
-        'wal_log_hints': ('on', _false_validator, 90400)
+        'wal_log_hints': ('on', _bool_is_true_validator, 90400)
     })
 
     _RECOVERY_PARAMETERS = CaseInsensitiveSet(recovery_parameters.keys())
@@ -367,6 +369,30 @@ class ConfigHandler(object):
             configuration.append('pg_ident.conf')
         return configuration
 
+    def set_file_permissions(self, filename: str) -> None:
+        """Set permissions of file *filename* according to the expected permissions if it resides under PGDATA.
+
+        .. note::
+            Do nothing if the file is not under PGDATA.
+
+        :param filename: path to a file which permissions might need to be adjusted.
+        """
+        if is_subpath(self._postgresql.data_dir, filename):
+            pg_perm.set_permissions_from_data_directory(self._postgresql.data_dir)
+            os.chmod(filename, pg_perm.file_create_mode)
+
+    @contextmanager
+    def config_writer(self, filename: str) -> Iterator[ConfigWriter]:
+        """Create :class:`ConfigWriter` object and set permissions on a *filename*.
+
+        :param filename: path to a config file.
+
+        :yields: :class:`ConfigWriter` object.
+        """
+        with ConfigWriter(filename) as writer:
+            yield writer
+        self.set_file_permissions(filename)
+
     def save_configuration_files(self, check_custom_bootstrap: bool = False) -> bool:
         """
             copy postgresql.conf to postgresql.conf.backup to be able to retrieve configuration files
@@ -380,6 +406,7 @@ class ConfigHandler(object):
                     backup_file = os.path.join(self._postgresql.data_dir, f + '.backup')
                     if os.path.isfile(config_file):
                         shutil.copy(config_file, backup_file)
+                        self.set_file_permissions(backup_file)
             except IOError:
                 logger.exception('unable to create backup copies of configuration files')
         return True
@@ -393,9 +420,11 @@ class ConfigHandler(object):
                 if not os.path.isfile(config_file):
                     if os.path.isfile(backup_file):
                         shutil.copy(backup_file, config_file)
+                        self.set_file_permissions(config_file)
                     # Previously we didn't backup pg_ident.conf, if file is missing just create empty
                     elif f == 'pg_ident.conf':
                         open(config_file, 'w').close()
+                        self.set_file_permissions(config_file)
         except IOError:
             logger.exception('unable to restore configuration files from backup')
 
@@ -409,7 +438,7 @@ class ConfigHandler(object):
         if self._postgresql.enforce_hot_standby_feedback:
             configuration['hot_standby_feedback'] = 'on'
 
-        with ConfigWriter(self._postgresql_conf) as f:
+        with self.config_writer(self._postgresql_conf) as f:
             include = self._config.get('custom_conf') or self._postgresql_base_conf_name
             f.writeline("include '{0}'\n".format(ConfigWriter.escape(include)))
             for name, value in sorted((configuration).items()):
@@ -439,6 +468,7 @@ class ConfigHandler(object):
         if not self.hba_file and not self._config.get('pg_hba'):
             with open(self._pg_hba_conf, 'a') as f:
                 f.write('\n{}\n'.format('\n'.join(config)))
+            self.set_file_permissions(self._pg_hba_conf)
         return True
 
     def replace_pg_hba(self) -> Optional[bool]:
@@ -458,14 +488,14 @@ class ConfigHandler(object):
                                   self.local_replication_address['host'], self.local_replication_address['port'],
                                   0, socket.SOCK_STREAM, socket.IPPROTO_TCP)})
 
-            with ConfigWriter(self._pg_hba_conf) as f:
+            with self.config_writer(self._pg_hba_conf) as f:
                 for address, t in addresses.items():
                     f.writeline((
                         '{0}\treplication\t{1}\t{3}\ttrust\n'
                         '{0}\tall\t{2}\t{3}\ttrust'
                     ).format(t, self.replication['username'], self._superuser.get('username') or 'all', address))
         elif not self.hba_file and self._config.get('pg_hba'):
-            with ConfigWriter(self._pg_hba_conf) as f:
+            with self.config_writer(self._pg_hba_conf) as f:
                 f.writelines(self._config['pg_hba'])
             return True
 
@@ -478,7 +508,7 @@ class ConfigHandler(object):
         """
 
         if not self.ident_file and self._config.get('pg_ident'):
-            with ConfigWriter(self._pg_ident_conf) as f:
+            with self.config_writer(self._pg_ident_conf) as f:
                 f.writelines(self._config['pg_ident'])
             return True
 
@@ -593,7 +623,24 @@ class ConfigHandler(object):
                                           'recovery_target_action', 'standby_mode', self._triggerfile_wrong_name})
         return CaseInsensitiveSet(self._RECOVERY_PARAMETERS - skip_params)
 
-    def _read_recovery_params(self) -> Tuple[Optional[CaseInsensitiveDict], Optional[bool]]:
+    def _read_recovery_params(self) -> Tuple[Optional[CaseInsensitiveDict], bool]:
+        """Read current recovery parameters values.
+
+        .. note::
+            We query Postgres only if we detected that Postgresql was restarted
+            or when at least one of the following files was updated:
+
+                * ``postgresql.conf``;
+                * ``postgresql.auto.conf``;
+                * ``passfile`` that is used in the ``primary_conninfo``.
+
+        :returns: a tuple with two elements:
+
+            * :class:`CaseInsensitiveDict` object with current values of recovery parameters,
+              or ``None`` if no configuration files were updated;
+
+            * ``True`` if new values of recovery parameters were queried, ``False`` otherwise.
+        """
         if self._postgresql.is_starting():
             return None, False
 
@@ -614,11 +661,20 @@ class ConfigHandler(object):
             self._postgresql_conf_mtime = pg_conf_mtime
             self._auto_conf_mtime = auto_conf_mtime
             self._postmaster_ctime = postmaster_ctime
-        except Exception:
+        except Exception as exc:
+            if all((isinstance(exc, PostgresConnectionException),
+                    self._postgresql_conf_mtime == pg_conf_mtime,
+                    self._auto_conf_mtime == auto_conf_mtime,
+                    self._passfile_mtime == passfile_mtime,
+                    self._postmaster_ctime != postmaster_ctime)):
+                # We detected that the connection to postgres fails, but the process creation time of the postmaster
+                # doesn't match the old value. It is an indicator that Postgres crashed and either doing crash
+                # recovery or down. In this case we return values like nothing changed in the config.
+                return None, False
             values = None
         return values, True
 
-    def _read_recovery_params_pre_v12(self) -> Tuple[Optional[CaseInsensitiveDict], Optional[bool]]:
+    def _read_recovery_params_pre_v12(self) -> Tuple[Optional[CaseInsensitiveDict], bool]:
         recovery_conf_mtime = mtime(self._recovery_conf)
         passfile_mtime = mtime(self._passfile) if self._passfile else False
         if recovery_conf_mtime == self._recovery_conf_mtime and passfile_mtime == self._passfile_mtime:
@@ -800,9 +856,11 @@ class ConfigHandler(object):
         if self._postgresql.major_version >= 120000:
             if parse_bool(recovery_params.pop('standby_mode', None)):
                 open(self._standby_signal, 'w').close()
+                self.set_file_permissions(self._standby_signal)
             else:
                 self._remove_file_if_exists(self._standby_signal)
                 open(self._recovery_signal, 'w').close()
+                self.set_file_permissions(self._recovery_signal)
 
             def restart_required(name: str) -> bool:
                 if self._postgresql.major_version >= 140000:
@@ -813,8 +871,7 @@ class ConfigHandler(object):
             self._current_recovery_params = CaseInsensitiveDict({n: [v, restart_required(n), self._postgresql_conf]
                                                                  for n, v in recovery_params.items()})
         else:
-            with ConfigWriter(self._recovery_conf) as f:
+            with self.config_writer(self._recovery_conf) as f:
-                os.chmod(self._recovery_conf, stat.S_IWRITE | stat.S_IREAD)
                 self._write_recovery_params(f, recovery_params)
 
     def remove_recovery_conf(self) -> None:
@@ -843,6 +900,7 @@ class ConfigHandler(object):
         if overwrite:
             try:
                 with open(self._auto_conf, 'w') as f:
+                    self.set_file_permissions(self._auto_conf)
                     for raw_line in lines:
                         f.write(raw_line)
             except Exception:
@@ -910,24 +968,32 @@ class ConfigHandler(object):
                 return 'localhost'  # connection via localhost is preferred
         return listen_addresses[0].strip()  # can't use localhost, take first address from listen_addresses
 
-    @property
-    def local_connect_kwargs(self) -> Dict[str, Any]:
-        ret = self._local_address.copy()
-        # add all of the other connection settings that are available
-        ret.update(self._superuser)
-        # if the "username" parameter is present, it actually needs to be "user"
-        # for connecting to PostgreSQL
-        if 'username' in self._superuser:
-            ret['user'] = self._superuser['username']
-            del ret['username']
-        # ensure certain Patroni configurations are available
-        ret.update({'dbname': self._postgresql.database,
-                    'fallback_application_name': 'Patroni',
-                    'connect_timeout': 3,
-                    'options': '-c statement_timeout=2000'})
-        return ret
-
     def resolve_connection_addresses(self) -> None:
+        """Calculates and sets local and remote connection urls and options.
+
+        This method sets:
+            * :attr:`Postgresql.connection_string <patroni.postgresql.Postgresql.connection_string>` attribute, which
+              is later written to the member key in DCS as ``conn_url``.
+            * :attr:`ConfigHandler.local_replication_address` attribute, which is used for replication connections to
+              local postgres.
+            * :attr:`ConnectionPool.conn_kwargs <patroni.postgresql.connection.ConnectionPool.conn_kwargs>` attribute,
+              which is used for superuser connections to local postgres.
+
+        .. note::
+            If there is a valid directory in ``postgresql.parameters.unix_socket_directories`` in the Patroni
+            configuration and ``postgresql.use_unix_socket`` and/or ``postgresql.use_unix_socket_repl``
+            are set to ``True``, we respectively use unix sockets for superuser and replication connections
+            to local postgres.
+
+            If there is a requirement to use unix sockets, but nothing is set in the
+            ``postgresql.parameters.unix_socket_directories``, we omit a ``host`` in connection parameters relying
+            on the ability of ``libpq`` to connect via some default unix socket directory.
+
+            If unix sockets are not requested we "switch" to TCP, prefering to use ``localhost`` if it is possible
+            to deduce that Postgres is listening on a local interface address.
+
+            Otherwise we just used the first address specified in the ``listen_addresses`` GUC.
+        """
         port = self._server_parameters['port']
         tcp_local_address = self._get_tcp_local_address()
         netloc = self._config.get('connect_address') or tcp_local_address + ':' + port
@@ -940,12 +1006,25 @@ class ConfigHandler(object):
 
         tcp_local_address = {'host': tcp_local_address, 'port': port}
 
-        self._local_address = unix_local_address if self._config.get('use_unix_socket') else tcp_local_address
         self.local_replication_address = unix_local_address\
             if self._config.get('use_unix_socket_repl') else tcp_local_address
 
         self._postgresql.connection_string = uri('postgres', netloc, self._postgresql.database)
-        self._postgresql.set_connection_kwargs(self.local_connect_kwargs)
+
+        local_address = unix_local_address if self._config.get('use_unix_socket') else tcp_local_address
+        local_conn_kwargs = {
+            **local_address,
+            **self._superuser,
+            'dbname': self._postgresql.database,
+            'fallback_application_name': 'Patroni',
+            'connect_timeout': 3,
+            'options': '-c statement_timeout=2000'
+        }
+        # if the "username" parameter is present, it actually needs to be "user" for connecting to PostgreSQL
+        if 'username' in local_conn_kwargs:
+            local_conn_kwargs['user'] = local_conn_kwargs.pop('username')
+        # "notify" connection_pool about the "new" local connection address
+        self._postgresql.connection_pool.conn_kwargs = local_conn_kwargs
 
     def _get_pg_settings(
             self, names: Collection[str]
@@ -1061,10 +1140,10 @@ class ConfigHandler(object):
             if self._postgresql.major_version >= 90500:
                 time.sleep(1)
                 try:
-                    pending_restart = (self._postgresql.query(
+                    pending_restart = self._postgresql.query(
                         'SELECT COUNT(*) FROM pg_catalog.pg_settings'
                         ' WHERE pg_catalog.lower(name) != ALL(%s) AND pending_restart',
-                        [n.lower() for n in self._RECOVERY_PARAMETERS]).fetchone() or (0,))[0] > 0
+                        [n.lower() for n in self._RECOVERY_PARAMETERS])[0][0] > 0
                     self._postgresql.set_pending_restart(pending_restart)
                 except Exception as e:
                     logger.warning('Exception %r when running query', e)

patroni/postgresql/connection.py

@@ -2,49 +2,157 @@ import logging
 
 from contextlib import contextmanager
 from threading import Lock
-from typing import Any, Dict, Generator, Union, TYPE_CHECKING
+from typing import Any, Dict, Iterator, List, Optional, Union, Tuple, TYPE_CHECKING
 if TYPE_CHECKING:  # pragma: no cover
-    from psycopg import Connection as Connection3, Cursor
+    from psycopg import Connection, Cursor
     from psycopg2 import connection, cursor
 
 from .. import psycopg
+from ..exceptions import PostgresConnectionException
 
 logger = logging.getLogger(__name__)
 
 
-class Connection(object):
+class NamedConnection:
+    """Helper class to manage ``psycopg`` connections from Patroni to PostgreSQL.
+
+    :ivar server_version: PostgreSQL version in integer format where we are connected to.
+    """
+
     server_version: int
 
-    def __init__(self) -> None:
+    def __init__(self, pool: 'ConnectionPool', name: str, kwargs_override: Optional[Dict[str, Any]]) -> None:
-        self._lock = Lock()
+        """Create an instance of :class:`NamedConnection` class.
+
+        :param pool: reference to a :class:`ConnectionPool` object.
+        :param name: name of the connection.
+        :param kwargs_override: :class:`dict` object with connection parameters that should be
+                                different from default values provided by connection *pool*.
+        """
+        self._pool = pool
+        self._name = name
+        self._kwargs_override = kwargs_override or {}
+        self._lock = Lock()  # used to make sure that only one connection to postgres is established
         self._connection = None
-        self._cursor_holder = None
 
-    def set_conn_kwargs(self, conn_kwargs: Dict[str, Any]) -> None:
+    @property
-        self._conn_kwargs = conn_kwargs
+    def _conn_kwargs(self) -> Dict[str, Any]:
+        """Connection parameters for this :class:`NamedConnection`."""
+        return {**self._pool.conn_kwargs, **self._kwargs_override, 'application_name': f'Patroni {self._name}'}
 
-    def get(self) -> Union['connection', 'Connection3[Any]']:
+    def get(self) -> Union['connection', 'Connection[Any]']:
+        """Get ``psycopg``/``psycopg2`` connection object.
+
+        .. note::
+            Opens a new connection if necessary.
+
+        :returns: ``psycopg`` or ``psycopg2`` connection object.
+        """
         with self._lock:
             if not self._connection or self._connection.closed != 0:
+                logger.info("establishing a new patroni %s connection to postgres", self._name)
                 self._connection = psycopg.connect(**self._conn_kwargs)
                 self.server_version = getattr(self._connection, 'server_version', 0)
         return self._connection
 
-    def cursor(self) -> Union['cursor', 'Cursor[Any]']:
+    def query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]:
-        if not self._cursor_holder or self._cursor_holder.closed or self._cursor_holder.connection.closed != 0:
+        """Execute a query with parameters and optionally returns a response.
-            logger.info("establishing a new patroni connection to the postgres cluster")
-            self._cursor_holder = self.get().cursor()
-        return self._cursor_holder
 
-    def close(self) -> None:
+        :param sql: SQL statement to execute.
+        :param params: parameters to pass.
+
+        :returns: a query response as a list of tuples if there is any.
+        :raises:
+            :exc:`~psycopg.Error` if had issues while executing *sql*.
+
+            :exc:`~patroni.exceptions.PostgresConnectionException`: if had issues while connecting to the database.
+        """
+        cursor = None
+        try:
+            with self.get().cursor() as cursor:
+                cursor.execute(sql.encode('utf-8'), params or None)
+                return cursor.fetchall() if cursor.rowcount and cursor.rowcount > 0 else []
+        except psycopg.Error as exc:
+            if cursor and cursor.connection.closed == 0:
+                # When connected via unix socket, psycopg2 can't recoginze 'connection lost' and leaves
+                # `self._connection.closed == 0`, but the generic exception is raised. It doesn't make
+                # sense to continue with existing connection and we will close it, to avoid its reuse.
+                if type(exc) in (psycopg.DatabaseError, psycopg.OperationalError):
+                    self.close()
+                else:
+                    raise exc
+            raise PostgresConnectionException('connection problems') from exc
+
+    def close(self, silent: bool = False) -> bool:
+        """Close the psycopg connection to postgres.
+
+        :param silent: whether the method should not write logs.
+
+        :returns: ``True`` if ``psycopg`` connection was closed, ``False`` otherwise.``
+        """
+        ret = False
         if self._connection and self._connection.closed == 0:
             self._connection.close()
-            logger.info("closed patroni connection to the postgresql cluster")
+            if not silent:
-        self._cursor_holder = self._connection = None
+                logger.info("closed patroni %s connection to postgres", self._name)
+            ret = True
+        self._connection = None
+        return ret
+
+
+class ConnectionPool:
+    """Helper class to manage named connections from Patroni to PostgreSQL.
+
+    The instance keeps named :class:`NamedConnection` objects and parameters that must be used for new connections.
+    """
+
+    def __init__(self) -> None:
+        """Create an instance of :class:`ConnectionPool` class."""
+        self._lock = Lock()
+        self._connections: Dict[str, NamedConnection] = {}
+        self._conn_kwargs: Dict[str, Any] = {}
+
+    @property
+    def conn_kwargs(self) -> Dict[str, Any]:
+        """Connection parameters that must be used for new ``psycopg`` connections."""
+        with self._lock:
+            return self._conn_kwargs.copy()
+
+    @conn_kwargs.setter
+    def conn_kwargs(self, value: Dict[str, Any]) -> None:
+        """Set new connection parameters.
+
+        :param value: :class:`dict` object with connection parameters.
+        """
+        with self._lock:
+            self._conn_kwargs = value
+
+    def get(self, name: str, kwargs_override: Optional[Dict[str, Any]] = None) -> NamedConnection:
+        """Get a new named :class:`NamedConnection` object from the pool.
+
+        .. note::
+            Creates a new :class:`NamedConnection` object if it doesn't yet exist in the pool.
+
+        :param name: name of the connection.
+        :param kwargs_override: :class:`dict` object with connection parameters that should be
+                                different from default values provided by :attr:`conn_kwargs`.
+
+        :returns: :class:`NamedConnection` object.
+        """
+        with self._lock:
+            if name not in self._connections:
+                self._connections[name] = NamedConnection(self, name, kwargs_override)
+        return self._connections[name]
+
+    def close(self) -> None:
+        """Close all named connections from Patroni to PostgreSQL registered in the pool."""
+        with self._lock:
+            if any(conn.close(True) for conn in self._connections.values()):
+                logger.info("closed patroni connections to postgres")
 
 
 @contextmanager
-def get_connection_cursor(**kwargs: Any) -> Generator[Union['cursor', 'Cursor[Any]'], None, None]:
+def get_connection_cursor(**kwargs: Any) -> Iterator[Union['cursor', 'Cursor[Any]']]:
     conn = psycopg.connect(**kwargs)
     with conn.cursor() as cur:
         yield cur

patroni/postgresql/rewind.py

@@ -280,7 +280,7 @@ class Rewind(object):
         """After promote issue a CHECKPOINT from a new thread and asynchronously check the result.
         In case if CHECKPOINT failed, just check that timeline in pg_control was updated."""
 
-        if self._state == REWIND_STATUS.INITIAL and self._postgresql.is_leader():
+        if self._state != REWIND_STATUS.CHECKPOINT and self._postgresql.is_primary():
             with self._checkpoint_task_lock:
                 if self._checkpoint_task:
                     with self._checkpoint_task:
@@ -370,7 +370,7 @@ class Rewind(object):
                 # it is the author of archive_command, who is responsible
                 # for not overriding the WALs already present in archive
                 logger.info('Trying to archive %s: %s', wal, cmd)
-                if self._postgresql.cancellable.call(shlex.split(cmd)) == 0:
+                if self._postgresql.cancellable.call([cmd], shell=True) == 0:
                     new_name = os.path.join(status_dir, wal + '.done')
                     try:
                         shutil.move(old_name, new_name)

patroni/postgresql/slots.py

@@ -1,15 +1,20 @@
+"""Replication slot handling.
+
+Provides classes for the creation, monitoring, management and synchronisation of PostgreSQL replication slots.
+"""
+
 import logging
 import os
 import shutil
-
 from collections import defaultdict
 from contextlib import contextmanager
 from threading import Condition, Thread
-from typing import Any, Dict, Generator, List, Optional, Union, Tuple, TYPE_CHECKING
+from typing import Any, Dict, Iterator, List, Optional, Union, Tuple, TYPE_CHECKING, Collection
 
 from .connection import get_connection_cursor
 from .misc import format_lsn, fsync_dir
 from ..dcs import Cluster, Leader
+from ..file_perm import pg_perm
 from ..psycopg import OperationalError
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -42,9 +47,17 @@ def compare_slots(s1: Dict[str, Any], s2: Dict[str, Any], dbid: str = 'database'
 
 
 class SlotsAdvanceThread(Thread):
+    """Daemon process :class:``Thread`` object for advancing logical replication slots on replicas.
+
+    This ensures that slot advancing queries sent to postgres do not block the main loop.
+    """
 
     def __init__(self, slots_handler: 'SlotsHandler') -> None:
-        super(SlotsAdvanceThread, self).__init__()
+        """Create and start a new thread for handling slot advance queries.
+
+        :param slots_handler: The calling class instance for reference to slot information attributes.
+        """
+        super().__init__()
         self.daemon = True
         self._slots_handler = slots_handler
 
@@ -58,6 +71,13 @@ class SlotsAdvanceThread(Thread):
         self.start()
 
     def sync_slot(self, cur: Union['cursor', 'Cursor[Any]'], database: str, slot: str, lsn: int) -> None:
+        """Execute a ``pg_replication_slot_advance`` query and store success for scheduled synchronisation task.
+
+        :param cur: database connection cursor.
+        :param database: name of the database associated with the slot.
+        :param slot: name of the slot to be synchronised.
+        :param lsn: last known LSN position
+        """
         failed = copy = False
         try:
             cur.execute("SELECT pg_catalog.pg_replication_slot_advance(%s, %s)", (slot, format_lsn(lsn)))
@@ -79,6 +99,11 @@ class SlotsAdvanceThread(Thread):
                     self._scheduled.pop(database)
 
     def sync_slots_in_database(self, database: str, slots: List[str]) -> None:
+        """Synchronise slots for a single database.
+
+        :param database: name of the database.
+        :param slots: list of slot names to synchronise.
+        """
         with self._slots_handler.get_local_connection_cursor(dbname=database, options='-c statement_timeout=0') as cur:
             for slot in slots:
                 with self._condition:
@@ -87,6 +112,7 @@ class SlotsAdvanceThread(Thread):
                     self.sync_slot(cur, database, slot, lsn)
 
     def sync_slots(self) -> None:
+        """Synchronise slots for all scheduled databases."""
         with self._condition:
             databases = list(self._scheduled.keys())
         for database in databases:
@@ -99,6 +125,12 @@ class SlotsAdvanceThread(Thread):
                     logger.error('Failed to advance replication slots in database %s: %r', database, e)
 
     def run(self) -> None:
+        """Thread main loop entrypoint.
+
+        .. note::
+            Thread will wait until a sync is scheduled from outside, normally triggered during the HA loop or a wakeup
+            call.
+        """
         while True:
             with self._condition:
                 if not self._scheduled:
@@ -107,6 +139,14 @@ class SlotsAdvanceThread(Thread):
             self.sync_slots()
 
     def schedule(self, advance_slots: Dict[str, Dict[str, int]]) -> Tuple[bool, List[str]]:
+        """Trigger a synchronisation of slots.
+
+        This is the main entrypoint for Patroni HA loop wakeup call.
+
+        :param advance_slots: dictionary containing slots that need to be advanced
+
+        :return: tuple of failure status and a list of slots to be copied
+        """
         with self._condition:
             for database, values in advance_slots.items():
                 self._scheduled[database].update(values)
@@ -118,40 +158,75 @@ class SlotsAdvanceThread(Thread):
         return ret
 
     def on_promote(self) -> None:
+        """Reset state of the daemon."""
         with self._condition:
             self._scheduled.clear()
             self._failed = False
             self._copy_slots = []
 
 
-class SlotsHandler(object):
+class SlotsHandler:
+    """Handler for managing and storing information on replication slots in PostgreSQL.
+
+    :ivar pg_replslot_dir: system location path of the PostgreSQL replication slots.
+    :ivar _logical_slots_processing_queue: yet to be processed logical replication slots on the primary
+    """
 
     def __init__(self, postgresql: 'Postgresql') -> None:
+        """Create an instance with storage attributes for replication slots and schedule the first synchronisation.
+
+        :param postgresql: Calling class instance providing interface to PostgreSQL.
+        """
+        self._force_readiness_check = False
+        self._schedule_load_slots = False
         self._postgresql = postgresql
         self._advance = None
         self._replication_slots: Dict[str, Dict[str, Any]] = {}  # already existing replication slots
-        self._unready_logical_slots: Dict[str, Optional[int]] = {}
+        self._logical_slots_processing_queue: Dict[str, Optional[int]] = {}
         self.pg_replslot_dir = os.path.join(self._postgresql.data_dir, 'pg_replslot')
         self.schedule()
 
-    def _query(self, sql: str, *params: Any) -> Union['cursor', 'Cursor[Any]']:
+    def _query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]:
+        """Helper method for :meth:`Postgresql.query`.
+
+        :param sql: SQL statement to execute.
+        :param params: parameters to pass through to :meth:`Postgresql.query`.
+
+        :returns: query response.
+        """
         return self._postgresql.query(sql, *params, retry=False)
 
     @staticmethod
-    def _copy_items(src: Dict[str, Any], dst: Dict[str, Any], keys: Optional[List[str]] = None) -> None:
+    def _copy_items(src: Dict[str, Any], dst: Dict[str, Any], keys: Optional[Collection[str]] = None) -> None:
+        """Select values from *src* dictionary to update in *dst* dictionary for optional supplied *keys*.
+
+        :param src: source dictionary that *keys* will be looked up from.
+        :param dst: destination dictionary to be updated.
+        :param keys: optional list of keys to be looked up in the source dictionary.
+        """
         dst.update({key: src[key] for key in keys or ('datoid', 'catalog_xmin', 'confirmed_flush_lsn')})
 
     def process_permanent_slots(self, slots: List[Dict[str, Any]]) -> Dict[str, int]:
-        """This methods solves three problems at once (I know, it is weird).
+        """Process replication slot information from the host and prepare information used in subsequent cluster tasks.
+
+        .. note::
+            This methods solves three problems.
+
+            The ``cluster_info_query`` from :class:``Postgresql`` is executed every HA loop and returns information
+            about all replication slots that exists on the current host.
+
+            Based on this information perform the following actions:
+
+            1. For the primary we want to expose to DCS permanent logical slots, therefore build (and return) a dict
+               that maps permanent logical slot names to ``confirmed_flush_lsn``.
+            2. detect if one of the previously known permanent slots is missing and schedule resync.
+            3. Update the local cache with the fresh ``catalog_xmin`` and ``confirmed_flush_lsn`` for every known slot.
 
-        The cluster_info_query from `Postgresql` is executed every HA loop and returns
-        information about all replication slots that exists on the current host.
-        Based on this information we perform the following actions:
-        1. For the primary we want to expose to DCS permanent logical slots, therefore the method
-           builds (and returns) a dict, that maps permanent logical slot names and confirmed_flush_lsns.
-        2. This method also detects if one of the previously known permanent slots got missing and schedules resync.
-        3. Updates the local cache with the fresh catalog_xmin and confirmed_flush_lsn for every known slot.
            This info is used when performing the check of logical slot readiness on standbys.
+
+        :param slots: replication slot information that exists on the current host.
+
+        :return: dictionary of logical slot names to ``confirmed_flush_lsn``.
         """
         ret: Dict[str, int] = {}
 
@@ -173,14 +248,23 @@ class SlotsHandler(object):
         return ret
 
     def load_replication_slots(self) -> None:
+        """Query replication slot information from the database and store it for processing by other tasks.
+
+        .. note::
+            Only supported from PostgreSQL version 9.4 onwards.
+
+        Store replication slot ``name``, ``type``, ``plugin``, ``database`` and ``datoid``.
+        If PostgreSQL version is 10 or newer also store ``catalog_xmin`` and ``confirmed_flush_lsn``.
+
+        When using logical slots, store information separately for slot synchronisation  on replica nodes.
+        """
         if self._postgresql.major_version >= 90400 and self._schedule_load_slots:
             replication_slots: Dict[str, Dict[str, Any]] = {}
-            extra = ", catalog_xmin, pg_catalog.pg_wal_lsn_diff(confirmed_flush_lsn, '0/0')::bigint"\
+            extra = ", catalog_xmin, pg_catalog.pg_wal_lsn_diff(confirmed_flush_lsn, '0/0')::bigint" \
                 if self._postgresql.major_version >= 100000 else ""
             skip_temp_slots = ' WHERE NOT temporary' if self._postgresql.major_version >= 100000 else ''
-            cursor = self._query('SELECT slot_name, slot_type, plugin, database, datoid'
+            for r in self._query('SELECT slot_name, slot_type, plugin, database, datoid'
-                                 '{0} FROM pg_catalog.pg_replication_slots{1}'.format(extra, skip_temp_slots))
+                                 f'{extra} FROM pg_catalog.pg_replication_slots{skip_temp_slots}'):
-            for r in cursor:
                 value = {'type': r[1]}
                 if r[1] == 'logical':
                     value.update(plugin=r[2], database=r[3], datoid=r[4])
@@ -190,33 +274,61 @@ class SlotsHandler(object):
             self._replication_slots = replication_slots
             self._schedule_load_slots = False
             if self._force_readiness_check:
-                self._unready_logical_slots = {n: None for n, v in replication_slots.items() if v['type'] == 'logical'}
+                self._logical_slots_processing_queue = {n: None for n, v in replication_slots.items()
+                                                        if v['type'] == 'logical'}
                 self._force_readiness_check = False
 
     def ignore_replication_slot(self, cluster: Cluster, name: str) -> bool:
+        """Check if slot *name* should not be managed by Patroni.
+
+        :param cluster: cluster state information object.
+        :param name: name of the slot to ignore
+
+        :returns: ``True`` if slot *name* matches any slot specified in ``ignore_slots`` configuration,
+                 otherwise will pass through and return result of :meth:`CitusHandler.ignore_replication_slot`.
+        """
         slot = self._replication_slots[name]
         if cluster.config:
             for matcher in cluster.config.ignore_slots_matchers:
-                if ((matcher.get("name") is None or matcher["name"] == name)
+                if (
-                   and all(not matcher.get(a) or matcher[a] == slot.get(a) for a in ('database', 'plugin', 'type'))):
+                        (matcher.get("name") is None or matcher["name"] == name)
+                        and all(not matcher.get(a) or matcher[a] == slot.get(a)
+                                for a in ('database', 'plugin', 'type'))
+                ):
                     return True
         return self._postgresql.citus_handler.ignore_replication_slot(slot)
 
     def drop_replication_slot(self, name: str) -> Tuple[bool, bool]:
-        """Returns a tuple(active, dropped)"""
+        """Drop a named slot from Postgres.
-        cursor = self._query(('WITH slots AS (SELECT slot_name, active'
+
-                              ' FROM pg_catalog.pg_replication_slots WHERE slot_name = %s),'
+        :param name: name of the slot to be dropped.
-                              ' dropped AS (SELECT pg_catalog.pg_drop_replication_slot(slot_name),'
+
-                              ' true AS dropped FROM slots WHERE not active) '
+        :returns: a tuple of ``active`` and ``dropped``. ``active`` is ``True`` if the slot is active,
-                              'SELECT active, COALESCE(dropped, false) FROM slots'
+                  ``dropped`` is ``True`` if the slot was successfully dropped. If the slot was not found return
-                              ' FULL OUTER JOIN dropped ON true'), name)
+                  ``False`` for both.
-        row = cursor.fetchone()
+        """
-        if not row:
+        rows = self._query(('WITH slots AS (SELECT slot_name, active'
-            row = (False, False)
+                            ' FROM pg_catalog.pg_replication_slots WHERE slot_name = %s),'
-        return row
+                            ' dropped AS (SELECT pg_catalog.pg_drop_replication_slot(slot_name),'
+                            ' true AS dropped FROM slots WHERE not active) '
+                            'SELECT active, COALESCE(dropped, false) FROM slots'
+                            ' FULL OUTER JOIN dropped ON true'), name)
+        return rows[0] if rows else (False, False)
 
     def _drop_incorrect_slots(self, cluster: Cluster, slots: Dict[str, Any], paused: bool) -> None:
-        # drop old replication slots which are not presented in desired slots
+        """Compare required slots and configured as permanent slots with those found, dropping extraneous ones.
+
+        .. note::
+            Slots that are not contained in *slots* will be dropped.
+            Slots can be filtered out with ``ignore_slots`` configuration.
+
+            Slots that have matching names but do not match attributes in *slots* will also be dropped.
+
+        :param cluster: cluster state information object.
+        :param slots: dictionary of desired slot names as keys with slot attributes as a dictionary value, if known.
+        :param paused: ``True`` if the patroni cluster is currently in a paused state.
+        """
+        # drop old replication slots which are not presented in desired slots.
         for name in set(self._replication_slots) - set(slots):
             if not paused and not self.ignore_replication_slot(cluster, name):
                 active, dropped = self.drop_replication_slot(name)
@@ -228,6 +340,8 @@ class SlotsHandler(object):
                         logger.debug("Unable to drop unknown replication slot '%s', slot is still active", name)
                     else:
                         logger.error("Failed to drop replication slot '%s'", name)
+
+        # drop slots with matching names but attributes that do not match, e.g. `plugin` or `database`.
         for name, value in slots.items():
             if name in self._replication_slots and not compare_slots(value, self._replication_slots[name]):
                 logger.info("Trying to drop replication slot '%s' because value is changing from %s to %s",
@@ -239,31 +353,56 @@ class SlotsHandler(object):
                     self._schedule_load_slots = True
 
     def _ensure_physical_slots(self, slots: Dict[str, Any]) -> None:
+        """Create any missing physical replication *slots*.
+
+        Any failures are logged and do not interrupt creation of all *slots*.
+
+        :param slots: A dictionary mapping slot name to slot attributes. This method only considers a slot
+                      if the value is a dictionary with the key ``type`` and a value of ``physical``.
+        """
         immediately_reserve = ', true' if self._postgresql.major_version >= 90600 else ''
         for name, value in slots.items():
             if name not in self._replication_slots and value['type'] == 'physical':
                 try:
-                    self._query(("SELECT pg_catalog.pg_create_physical_replication_slot(%s{0})"
+                    self._query(f"SELECT pg_catalog.pg_create_physical_replication_slot(%s{immediately_reserve})"
-                                 " WHERE NOT EXISTS (SELECT 1 FROM pg_catalog.pg_replication_slots"
+                                f" WHERE NOT EXISTS (SELECT 1 FROM pg_catalog.pg_replication_slots"
-                                 " WHERE slot_type = 'physical' AND slot_name = %s)").format(
+                                f" WHERE slot_type = 'physical' AND slot_name = %s)",
-                                     immediately_reserve), name, name)
+                                name, name)
                 except Exception:
                     logger.exception("Failed to create physical replication slot '%s'", name)
                 self._schedule_load_slots = True
 
     @contextmanager
-    def get_local_connection_cursor(self, **kwargs: Any) -> Generator[Union['cursor', 'Cursor[Any]'], None, None]:
+    def get_local_connection_cursor(self, **kwargs: Any) -> Iterator[Union['cursor', 'Cursor[Any]']]:
-        conn_kwargs = self._postgresql.config.local_connect_kwargs
+        """Create a new database connection to local server.
-        conn_kwargs.update(kwargs)
+
+        Create a non-blocking connection cursor to avoid the situation where an execution of the query of
+        ``pg_replication_slot_advance`` takes longer than the timeout on a HA loop, which could cause a false
+        failure state.
+
+        :param kwargs: Any keyword arguments to pass to :func:`psycopg.connect`.
+
+        :yields: connection cursor object, note implementation varies depending on version of :mod:`psycopg`.
+        """
+        conn_kwargs = {**self._postgresql.connection_pool.conn_kwargs, **kwargs}
         with get_connection_cursor(**conn_kwargs) as cur:
             yield cur
 
     def _ensure_logical_slots_primary(self, slots: Dict[str, Any]) -> None:
+        """Create any missing logical replication *slots* on the primary.
+
+        If the logical slot already exists, copy state information into the replication slots structure stored in the
+        class instance.
+
+        :param slots: Slots that should exist are supplied in a dictionary, mapping slot name to any attributes.
+                      The method will only consider slots that have a value that is a dictionary with a key ``type``
+                      with a value that is ``logical``.
+
+        """
         # Group logical slots to be created by database name
         logical_slots: Dict[str, Dict[str, Dict[str, Any]]] = defaultdict(dict)
         for name, value in slots.items():
             if value['type'] == 'logical':
-                # If the logical already exists, copy some information about it into the original structure
                 if self._replication_slots.get(name, {}).get('datoid'):
                     self._copy_items(self._replication_slots[name], value)
                 else:
@@ -285,27 +424,56 @@ class SlotsHandler(object):
                     self._schedule_load_slots = True
 
     def schedule_advance_slots(self, slots: Dict[str, Dict[str, int]]) -> Tuple[bool, List[str]]:
+        """Wrapper to ensure slots advance daemon thread is started if not already.
+
+        :param slots: dictionary containing slot information.
+
+        :return: tuple with the result of the scheduling of slot advancement: ``failed`` and list of slots to copy.
+        """
         if not self._advance:
             self._advance = SlotsAdvanceThread(self)
         return self._advance.schedule(slots)
 
-    def _ensure_logical_slots_replica(self, cluster: Cluster, slots: Dict[str, Any]) -> List[str]:
+    def _ensure_logical_slots_replica(self, slots: Dict[str, Any]) -> List[str]:
+        """Update logical *slots* on replicas.
+
+        If the logical slot already exists, copy state information into the replication slots structure stored in the
+        class instance. Slots that exist are also advanced if their ``confirmed_flush_lsn`` is greater than the stored
+        state of the slot.
+
+        As logical slots can only be created when the primary is available, pass the list of slots that need to be
+        copied back to the caller. They will be created on replicas with :meth:`SlotsHandler.copy_logical_slots`.
+
+        :param slots: A dictionary mapping slot name to slot attributes. This method only considers a slot
+                      if the value is a dictionary with the key ``type`` and a value of ``logical``.
+
+        :returns: list of slots to be copied from the primary.
+        """
         # Group logical slots to be advanced by database name
         advance_slots: Dict[str, Dict[str, int]] = defaultdict(dict)
-        create_slots: List[str] = []  # And collect logical slots to be created on the replica
+        create_slots: List[str] = []  # Collect logical slots to be created on the replica
+
         for name, value in slots.items():
-            if value['type'] == 'logical':
+            if value['type'] != 'logical':
-                # If the logical already exists, copy some information about it into the original structure
+                continue
-                if self._replication_slots.get(name, {}).get('datoid'):
+
-                    self._copy_items(self._replication_slots[name], value)
+            # If the logical already exists, copy some information about it into the original structure
-                    if cluster.slots and name in cluster.slots:
+            if name in self._replication_slots and compare_slots(value, self._replication_slots[name]):
-                        try:  # Skip slots that doesn't need to be advanced
+                self._copy_items(self._replication_slots[name], value)
-                            if value['confirmed_flush_lsn'] < int(cluster.slots[name]):
+                if 'lsn' in value:  # The slot has feedback in DCS
-                                advance_slots[value['database']][name] = int(cluster.slots[name])
+                    try:  # Skip slots that don't need to be advanced
-                        except Exception as e:
+                        if value['confirmed_flush_lsn'] < int(value['lsn']):
-                            logger.error('Failed to parse "%s": %r', cluster.slots[name], e)
+                            advance_slots[value['database']][name] = int(value['lsn'])
-                elif cluster.slots and name in cluster.slots:  # We want to copy only slots with feedback in a DCS
+                    except Exception as e:
-                    create_slots.append(name)
+                        logger.error('Failed to parse "%s": %r', value['lsn'], e)
+            elif name not in self._replication_slots and 'lsn' in value:
+                # We want to copy only slots with feedback in a DCS
+                create_slots.append(name)
+
+        # Slots to be copied from the primary should be removed from the *slots* structure,
+        # otherwise Patroni falsely assumes that they already exist.
+        for name in create_slots:
+            slots.pop(name)
 
         error, copy_slots = self.schedule_advance_slots(advance_slots)
         if error:
@@ -314,25 +482,39 @@ class SlotsHandler(object):
 
     def sync_replication_slots(self, cluster: Cluster, nofailover: bool,
                                replicatefrom: Optional[str] = None, paused: bool = False) -> List[str]:
+        """During the HA loop read, check and alter replication slots found in the cluster.
+
+        Read physical and logical slots found on the primary, then compare to those configured in the DCS.
+        Drop any slots that do not match those required by configuration and are not configured as permanent.
+        Create any missing physical slots. If we are the leader then logical slots too, otherwise if logical slots
+        are known and active create them on replica nodes.
+
+        :param cluster: object containing stateful information for the cluster.
+        :param nofailover: ``True`` if this node has been tagged to not be a failover candidate.
+        :param replicatefrom: the tag containing the node to replicate from.
+        :param paused: ``True`` if the cluster is in maintenance mode.
+
+        :returns: list of logical replication slots names that should be copied from the primary.
+        """
         ret = []
-        if self._postgresql.major_version >= 90400 and cluster.config:
+        if self._postgresql.major_version >= 90400 and self._postgresql.global_config and cluster.config:
             try:
                 self.load_replication_slots()
 
-                slots = cluster.get_replication_slots(self._postgresql.name, self._postgresql.role,
+                slots = cluster.get_replication_slots(
-                                                      nofailover, self._postgresql.major_version, True)
+                    self._postgresql.name, self._postgresql.role, nofailover, self._postgresql.major_version,
+                    is_standby_cluster=self._postgresql.global_config.is_standby_cluster, show_error=True)
 
                 self._drop_incorrect_slots(cluster, slots, paused)
 
                 self._ensure_physical_slots(slots)
 
-                if self._postgresql.is_leader():
+                if self._postgresql.is_primary():
-                    self._unready_logical_slots.clear()
+                    self._logical_slots_processing_queue.clear()
                     self._ensure_logical_slots_primary(slots)
-                elif cluster.slots and slots:
+                else:
-                    self.check_logical_slots_readiness(cluster, nofailover, replicatefrom)
+                    self.check_logical_slots_readiness(cluster, replicatefrom)
-
+                    ret = self._ensure_logical_slots_replica(slots)
-                    ret = self._ensure_logical_slots_replica(cluster, slots)
 
                 self._replication_slots = slots
             except Exception:
@@ -341,57 +523,121 @@ class SlotsHandler(object):
         return ret
 
     @contextmanager
-    def _get_leader_connection_cursor(self, leader: Leader) -> Generator[Union['cursor', 'Cursor[Any]'], None, None]:
+    def _get_leader_connection_cursor(self, leader: Leader) -> Iterator[Union['cursor', 'Cursor[Any]']]:
+        """Create a new database connection to the leader.
+
+        .. note::
+            Uses rewind user credentials because it has enough permissions to read files from PGDATA.
+            Sets the options ``connect_timeout`` to ``3`` and ``statement_timeout`` to ``2000``.
+
+        :param leader: object with information on the leader
+
+        :yields: connection cursor object, note implementation varies depending on version of ``psycopg``.
+        """
         conn_kwargs = leader.conn_kwargs(self._postgresql.config.rewind_credentials)
         conn_kwargs['dbname'] = self._postgresql.database
         with get_connection_cursor(connect_timeout=3, options="-c statement_timeout=2000", **conn_kwargs) as cur:
             yield cur
 
-    def check_logical_slots_readiness(self, cluster: Cluster, nofailover: bool, replicatefrom: Optional[str]) -> None:
+    def check_logical_slots_readiness(self, cluster: Cluster, replicatefrom: Optional[str]) -> bool:
+        """Determine whether all known logical slots are synchronised from the leader.
+
+        1) Retrieve the current ``catalog_xmin`` value for the physical slot from the cluster leader, and
+        2) using previously stored list of "unready" logical slots, those which have yet to be checked hence have no
+           stored slot attributes,
+        3) store logical slot ``catalog_xmin`` when the physical slot ``catalog_xmin`` becomes valid.
+
+        :param cluster: object containing stateful information for the cluster.
+        :param replicatefrom: name of the member that should be used to replicate from.
+
+        :returns: ``False`` if any issue while checking logical slots readiness, ``True`` otherwise.
+        """
         catalog_xmin = None
-        if self._unready_logical_slots and cluster.leader:
+        if self._logical_slots_processing_queue and cluster.leader:
             slot_name = cluster.get_my_slot_name_on_primary(self._postgresql.name, replicatefrom)
             try:
                 with self._get_leader_connection_cursor(cluster.leader) as cur:
                     cur.execute("SELECT slot_name, catalog_xmin FROM pg_catalog.pg_get_replication_slots()"
                                 " WHERE NOT pg_catalog.pg_is_in_recovery() AND slot_name = ANY(%s)",
-                                ([n for n, v in self._unready_logical_slots.items() if v is None] + [slot_name],))
+                                ([n for n, v in self._logical_slots_processing_queue.items()
+                                  if v is None] + [slot_name],))
                     slots = {row[0]: row[1] for row in cur}
                     if slot_name not in slots:
-                        return logger.warning('Physical slot %s does not exist on the primary', slot_name)
+                        logger.warning('Physical slot %s does not exist on the primary', slot_name)
+                        return False
                     catalog_xmin = slots.pop(slot_name)
             except Exception as e:
-                return logger.error("Failed to check %s physical slot on the primary: %r", slot_name, e)
+                logger.error("Failed to check %s physical slot on the primary: %r", slot_name, e)
-            # Remember catalog_xmin of logical slots on the primary when catalog_xmin of
+                return False
-            # the physical slot became valid. Logical slots on replica will be safe to use after
-            # promote when catalog_xmin of the physical slot overtakes these values.
-            if catalog_xmin is not None:
-                for name, value in slots.items():
-                    self._unready_logical_slots[name] = value
-            else:  # Replica isn't streaming or the hot_standby_feedback isn't enabled
-                try:
-                    cur = self._query("SELECT pg_catalog.current_setting('hot_standby_feedback')::boolean")
-                    row = cur.fetchone()
-                    if row and not row[0]:
-                        logger.error('Logical slot failover requires "hot_standby_feedback".'
-                                     ' Please check postgresql.auto.conf')
-                except Exception as e:
-                    logger.error('Failed to check the hot_standby_feedback setting: %r', e)
-                return  # since `catalog_xmin` isn't valid further checks don't make any sense
 
-        for name in list(self._unready_logical_slots):
+            if not self._update_pending_logical_slot_primary(slots, catalog_xmin):
-            value = self._replication_slots.get(name)
+                return False  # since `catalog_xmin` isn't valid further checks don't make any sense
-            # The logical slot on a replica is safe to use when the physical replica slot on the primary:
+
-            # 1. has a nonzero/non-null catalog_xmin
+        self._ready_logical_slots(catalog_xmin)
-            # 2. has a catalog_xmin that is not newer (greater) than the catalog_xmin of any slot on the standby
+        return True
-            # 3. overtook the catalog_xmin of remembered values of logical slots on the primary.
+
-            if not value or catalog_xmin is not None and\
+    def _update_pending_logical_slot_primary(self, slots: Dict[str, Any], catalog_xmin: Optional[int] = None) -> bool:
-                    self._unready_logical_slots[name] <= catalog_xmin <= value['catalog_xmin']:
+        """Store pending logical slot information for ``catalog_xmin`` on the primary.
-                del self._unready_logical_slots[name]
+
-                if value:
+        Remember ``catalog_xmin`` of logical slots on the primary when ``catalog_xmin`` of the physical slot became
+        valid. Logical slots on replica will be safe to use after promote when ``catalog_xmin`` of the physical slot
+        overtakes these values.
+
+        :param slots: dictionary of slot information from the primary
+        :param catalog_xmin: ``catalog_xmin`` of the physical slot used by this replica to stream changes from primary.
+
+        :returns: ``False`` if any issue was faced while processing, ``True`` otherwise.
+        """
+        if catalog_xmin is not None:
+            for name, value in slots.items():
+                self._logical_slots_processing_queue[name] = value
+            return True
+
+        # Replica isn't streaming or the hot_standby_feedback isn't enabled
+        try:
+            if not self._query("SELECT pg_catalog.current_setting('hot_standby_feedback')::boolean")[0][0]:
+                logger.error('Logical slot failover requires "hot_standby_feedback". Please check postgresql.auto.conf')
+        except Exception as e:
+            logger.error('Failed to check the hot_standby_feedback setting: %r', e)
+        return False
+
+    def _ready_logical_slots(self, primary_physical_catalog_xmin: Optional[int] = None) -> None:
+        """Ready logical slots by comparing primary physical slot ``catalog_xmin`` to logical ``catalog_xmin``.
+
+        The logical slot on a replica is safe to use when the physical replica slot on the primary:
+
+            1. has a nonzero/non-null ``catalog_xmin`` represented by ``primary_physical_xmin``.
+            2. has a ``catalog_xmin`` that is not newer (greater) than the ``catalog_xmin`` of any slot on the standby
+            3. overtook the ``catalog_xmin`` of remembered values of logical slots on the primary.
+
+        :param primary_physical_catalog_xmin: is the value retrieved from ``pg_catalog.pg_get_replication_slots()`` for
+                                              the physical replication slot on the primary.
+        """
+        # Make a copy of processing queue keys as a list as the queue dictionary is modified inside the loop.
+        for name in list(self._logical_slots_processing_queue):
+            primary_logical_catalog_xmin = self._logical_slots_processing_queue[name]
+            standby_logical_slot = self._replication_slots.get(name, {})
+            standby_logical_catalog_xmin = standby_logical_slot.get('catalog_xmin', 0)
+            if TYPE_CHECKING:  # pragma: no cover
+                assert primary_logical_catalog_xmin is not None
+
+            if (
+                    not standby_logical_slot
+                    or primary_physical_catalog_xmin is not None
+                    and primary_logical_catalog_xmin <= primary_physical_catalog_xmin <= standby_logical_catalog_xmin
+            ):
+
+                del self._logical_slots_processing_queue[name]
+
+                if standby_logical_slot:
                     logger.info('Logical slot %s is safe to be used after a failover', name)
 
     def copy_logical_slots(self, cluster: Cluster, create_slots: List[str]) -> None:
+        """Create logical replication slots on standby nodes.
+
+        :param cluster: object containing stateful information for the cluster.
+        :param create_slots: list of slot names to copy from the primary.
+        """
         leader = cluster.leader
         if not leader:
             return
@@ -418,34 +664,51 @@ class SlotsHandler(object):
                 logger.error("Failed to copy logical slots from the %s via postgresql connection: %r", leader.name, e)
 
         if copy_slots and self._postgresql.stop():
+            pg_perm.set_permissions_from_data_directory(self._postgresql.data_dir)
             for name, value in copy_slots.items():
-                slot_dir = os.path.join(self._postgresql.slots_handler.pg_replslot_dir, name)
+                slot_dir = os.path.join(self.pg_replslot_dir, name)
                 slot_tmp_dir = slot_dir + '.tmp'
                 if os.path.exists(slot_tmp_dir):
                     shutil.rmtree(slot_tmp_dir)
                 os.makedirs(slot_tmp_dir)
+                os.chmod(slot_tmp_dir, pg_perm.dir_create_mode)
                 fsync_dir(slot_tmp_dir)
-                with open(os.path.join(slot_tmp_dir, 'state'), 'wb') as f:
+                slot_filename = os.path.join(slot_tmp_dir, 'state')
+                with open(slot_filename, 'wb') as f:
+                    os.chmod(slot_filename, pg_perm.file_create_mode)
                     f.write(value['data'])
                     f.flush()
                     os.fsync(f.fileno())
                 if os.path.exists(slot_dir):
                     shutil.rmtree(slot_dir)
                 os.rename(slot_tmp_dir, slot_dir)
+                os.chmod(slot_dir, pg_perm.dir_create_mode)
                 fsync_dir(slot_dir)
-                self._unready_logical_slots[name] = None
+                self._logical_slots_processing_queue[name] = None
-            fsync_dir(self._postgresql.slots_handler.pg_replslot_dir)
+            fsync_dir(self.pg_replslot_dir)
             self._postgresql.start()
 
     def schedule(self, value: Optional[bool] = None) -> None:
+        """Schedule the loading of slot information from the database.
+
+        :param value: the optional value can be used to unschedule if set to ``False`` or force it to be ``True``.
+                      If it is omitted the value will be ``True`` if this PostgreSQL node supports slot replication.
+        """
         if value is None:
             value = self._postgresql.major_version >= 90400
         self._schedule_load_slots = self._force_readiness_check = value
 
     def on_promote(self) -> None:
+        """Entry point from HA cycle used when a standby node is to be promoted to primary.
+
+        .. note::
+            If logical replication slot synchronisation is enabled then slot advancement will be triggered.
+            If any logical slots that were copied are yet to be confirmed as ready a warning message will be logged.
+
+        """
         if self._advance:
             self._advance.on_promote()
 
-        if self._unready_logical_slots:
+        if self._logical_slots_processing_queue:
             logger.warning('Logical replication slots that might be unsafe to use after promote: %s',
-                           set(self._unready_logical_slots))
+                           set(self._logical_slots_processing_queue))

patroni/postgresql/sync.py

@@ -153,6 +153,72 @@ def parse_sync_standby_names(value: str) -> _SSN:
     return _SSN(sync_type, has_star, num, members)
 
 
+class _Replica(NamedTuple):
+    """Class representing a single replica that is eligible to be synchronous.
+
+    Attributes are taken from ``pg_stat_replication`` view and respective ``Cluster.members``.
+
+    :ivar pid: PID of walsender process.
+    :ivar application_name: matches with the ``Member.name``.
+    :ivar sync_state: possible values are: ``async``, ``potential``, ``quorum``, and ``sync``.
+    :ivar lsn: ``write_lsn``, ``flush_lsn``, or ``replay_lsn``, depending on the value of ``synchronous_commit`` GUC.
+    :ivar nofailover: whether the corresponding member has ``nofailover`` tag set to ``True``.
+    """
+    pid: int
+    application_name: str
+    sync_state: str
+    lsn: int
+    nofailover: bool
+
+
+class _ReplicaList(List[_Replica]):
+    """A collection of :class:``_Replica`` objects.
+
+    Values are reverse ordered by ``_Replica.sync_state`` and ``_Replica.lsn``.
+    That is, first there will be replicas that have ``sync_state`` == ``sync``, even if they are not
+    the most up-to-date in term of write/flush/replay LSN. It helps to keep the result of chosing new
+    synchronous nodes consistent in case if a synchronous standby member is slowed down OR async node
+    is receiving changes faster than the sync member. Such cases would trigger sync standby member
+    swapping, but only if lag on this member is exceeding a threshold (``maximum_lag_on_syncnode``).
+
+    :ivar max_lsn: maximum value of ``_Replica.lsn`` among all values. In case if there is just one
+                   element in the list we take value of ``pg_current_wal_flush_lsn()``.
+    """
+
+    def __init__(self, postgresql: 'Postgresql', cluster: Cluster) -> None:
+        """Create :class:``_ReplicaList`` object.
+
+        :param postgresql: reference to :class:``Postgresql`` object.
+        :param cluster: currently known cluster state from DCS.
+        """
+        super().__init__()
+
+        # We want to prioritize candidates based on `write_lsn``, ``flush_lsn``, or ``replay_lsn``.
+        # Which column exactly to pick depends on the values of ``synchronous_commit`` GUC.
+        sort_col = {
+            'remote_apply': 'replay',
+            'remote_write': 'write'
+        }.get(postgresql.synchronous_commit(), 'flush') + '_lsn'
+
+        members = CaseInsensitiveDict({m.name: m for m in cluster.members})
+        for row in postgresql.pg_stat_replication():
+            member = members.get(row['application_name'])
+
+            # We want to consider only rows from ``pg_stat_replication` that:
+            # 1. are known to be streaming (write/flush/replay LSN are not NULL).
+            # 2. can be mapped to a ``Member`` of the ``Cluster``:
+            #   a. ``Member`` doesn't have ``nosync`` tag set;
+            #   b. PostgreSQL on the member is known to be running and accepting client connections.
+            if member and row[sort_col] is not None and member.is_running and not member.nosync:
+                self.append(_Replica(row['pid'], row['application_name'],
+                                     row['sync_state'], row[sort_col], bool(member.nofailover)))
+
+        # Prefer replicas that are in state ``sync`` and with higher values of ``write``/``flush``/``replay`` LSN.
+        self.sort(key=lambda r: (r.sync_state, r.lsn), reverse=True)
+
+        self.max_lsn = max(self, key=lambda x: x.lsn).lsn if len(self) > 1 else postgresql.last_operation()
+
+
 class SyncHandler(object):
     """Class responsible for working with the `synchronous_standby_names`.
 
@@ -201,13 +267,29 @@ BEGIN
 END;$$""")
         self._postgresql.reset_cluster_info_state(None)  # Reset internal cache to query fresh values
 
+    def _process_replica_readiness(self, cluster: Cluster, replica_list: _ReplicaList) -> None:
+        """Flags replicas as truly "synchronous" when they have caught up with ``_primary_flush_lsn``.
+
+        :param cluster: current cluster topology from DCS
+        :param replica_list: collection of replicas that we want to evaluate.
+        """
+        for replica in replica_list:
+            # if standby name is listed in the /sync key we can count it as synchronous, otherwise
+            # it becomes really synchronous when sync_state = 'sync' and it is known that it managed to catch up
+            if replica.application_name not in self._ready_replicas\
+                    and replica.application_name in self._ssn_data.members\
+                    and (cluster.sync.matches(replica.application_name)
+                         or replica.sync_state == 'sync' and replica.lsn >= self._primary_flush_lsn):
+                self._ready_replicas[replica.application_name] = replica.pid
+
     def current_state(self, cluster: Cluster) -> Tuple[CaseInsensitiveSet, CaseInsensitiveSet]:
-        """Finds best candidates to be the synchronous standbys.
+        """Find the best candidates to be the synchronous standbys.
 
         Current synchronous standby is always preferred, unless it has disconnected or does not want to be a
         synchronous standby any longer.
 
         Standbys are selected based on values from the global configuration:
+
         - `maximum_lag_on_syncnode`: would help swapping unhealthy sync replica in case if it stops
           responding (or hung). Please set the value high enough so it won't unncessarily swap sync
           standbys during high loads. Any value less or equal of 0 keeps the behavior backward compatible.
@@ -218,31 +300,8 @@ END;$$""")
         """
         self._handle_synchronous_standby_names_change()
 
-        # Pick candidates based on who has higher replay/remote_write/flush lsn.
+        replica_list = _ReplicaList(self._postgresql, cluster)
-        sort_col = {
+        self._process_replica_readiness(cluster, replica_list)
-            'remote_apply': 'replay',
-            'remote_write': 'write'
-        }.get(self._postgresql.synchronous_commit(), 'flush') + '_lsn'
-
-        pg_stat_replication = [(r['pid'], r['application_name'], r['sync_state'], r[sort_col])
-                               for r in self._postgresql.pg_stat_replication()
-                               if r[sort_col] is not None]
-
-        members = CaseInsensitiveDict({m.name: m for m in cluster.members})
-        replica_list: List[Tuple[int, str, str, int, bool]] = []
-        # pg_stat_replication.sync_state has 4 possible states - async, potential, quorum, sync.
-        # That is, alphabetically they are in the reversed order of priority.
-        # Since we are doing reversed sort on (sync_state, lsn) tuples, it helps to keep the result
-        # consistent in case if a synchronous standby member is slowed down OR async node receiving
-        # changes faster than the sync member (very rare but possible).
-        # Such cases would trigger sync standby member swapping, but only if lag on a sync node exceeding a threshold.
-        for pid, app_name, sync_state, replica_lsn in sorted(pg_stat_replication, key=lambda r: r[2:4], reverse=True):
-            member = members.get(app_name)
-            if member and member.is_running and not member.tags.get('nosync', False):
-                replica_list.append((pid, member.name, sync_state, replica_lsn, bool(member.nofailover)))
-
-        max_lsn = max(replica_list, key=lambda x: x[3])[3]\
-            if len(replica_list) > 1 else self._postgresql.last_operation()
 
         if TYPE_CHECKING:  # pragma: no cover
             assert self._postgresql.global_config is not None
@@ -253,17 +312,11 @@ END;$$""")
         candidates = CaseInsensitiveSet()
         sync_nodes = CaseInsensitiveSet()
         # Prefer members without nofailover tag. We are relying on the fact that sorts are guaranteed to be stable.
-        for pid, app_name, sync_state, replica_lsn, _ in sorted(replica_list, key=lambda x: x[4]):
+        for replica in sorted(replica_list, key=lambda x: x.nofailover):
-            # if standby name is listed in the /sync key we can count it as synchronous, otherwice
+            if sync_node_maxlag <= 0 or replica_list.max_lsn - replica.lsn <= sync_node_maxlag:
-            # it becomes really synchronous when sync_state = 'sync' and it is known that it managed to catch up
+                candidates.add(replica.application_name)
-            if app_name not in self._ready_replicas and app_name in self._ssn_data.members and\
+                if replica.sync_state == 'sync' and replica.application_name in self._ready_replicas:
-                    (cluster.sync.matches(app_name) or sync_state == 'sync' and replica_lsn >= self._primary_flush_lsn):
+                    sync_nodes.add(replica.application_name)
-                self._ready_replicas[app_name] = pid
-
-            if sync_node_maxlag <= 0 or max_lsn - replica_lsn <= sync_node_maxlag:
-                candidates.add(app_name)
-                if sync_state == 'sync' and app_name in self._ready_replicas:
-                    sync_nodes.add(app_name)
             if len(candidates) >= sync_node_count:
                 break
 
@@ -286,7 +339,7 @@ END;$$""")
             sync_param = next(iter(sync), None)
 
         if not (self._postgresql.config.set_synchronous_standby_names(sync_param)
-                and self._postgresql.state == 'running' and self._postgresql.is_leader()) or has_asterisk:
+                and self._postgresql.state == 'running' and self._postgresql.is_primary()) or has_asterisk:
             return
 
         time.sleep(0.1)  # Usualy it takes 1ms to reload postgresql.conf, but we will give it 100ms

patroni/postgresql/validator.py

@@ -178,10 +178,11 @@ class ValidatorFactory:
 
         :returns: the Patroni validator object that corresponds to the specification found in *validator*.
 
-        :raises :class:`ValidatorFactoryNoType`: if *validator* contains no ``type`` key.
+        :raises:
-        :raises :class:`ValidatorFactoryInvalidType`: if ``type`` key from *validator* contains an invalid value.
+            :class:`ValidatorFactoryNoType`: if *validator* contains no ``type`` key.
-        :raises :class:`ValidatorFactoryInvalidSpec`: if *validator* contains an invalid set of attributes for the
+            :class:`ValidatorFactoryInvalidType`: if ``type`` key from *validator* contains an invalid value.
-            given ``type``.
+            :class:`ValidatorFactoryInvalidSpec`: if *validator* contains an invalid set of attributes for the given
+                ``type``.
 
         :Example:
 
@@ -265,7 +266,8 @@ def _read_postgres_gucs_validators_file(file: str) -> Dict[str, Any]:
     :returns: the YAML content parsed into a Python object. If any issue is faced while reading/parsing the file, then
         return ``None``.
 
-    :raises :class:`InvalidGucValidatorsFile`: if faces an issue while reading or parsing *file*.
+    :raises:
+        :class:`InvalidGucValidatorsFile`: if faces an issue while reading or parsing *file*.
     """
     try:
         with open(file, encoding='UTF-8') as stream:
@@ -288,7 +290,7 @@ def _load_postgres_gucs_validators() -> None:
     Any problem faced while reading or parsing files will be logged as a ``WARNING`` by the child function, and the
     corresponding file or validator will be ignored.
 
-    By default Patroni only ships the file ``0_postgres.yml``, which contains Community Postgres GUCs validators, but
+    By default, Patroni only ships the file ``0_postgres.yml``, which contains Community Postgres GUCs validators, but
     that behavior can be extended. For example: if a vendor wants to add GUC validators to Patroni for covering a custom
     Postgres build, then they can create their custom YAML files under ``available_parameters`` directory.
 
@@ -298,8 +300,10 @@ def _load_postgres_gucs_validators() -> None:
             writes them to ``postgresql.conf`` if running PG 12 and above).
 
     Then, each of these sections, if specified, may contain one or more attributes with the following structure:
+
         * key: the name of a GUC;
         * value: a list of validators. Each item in the list must contain a ``type`` attribute, which must be one among:
+
             * ``Bool``; or
             * ``Integer``; or
             * ``Real``; or
@@ -311,6 +315,7 @@ def _load_postgres_gucs_validators() -> None:
             class in this module.
 
     .. seealso::
+
         * :class:`Bool`;
         * :class:`Integer`;
         * :class:`Real`;
@@ -323,61 +328,62 @@ def _load_postgres_gucs_validators() -> None:
         This is a sample content for an YAML file based on Postgres GUCs, showing each of the supported types and
         sections:
 
-        ```yaml
+            .. code-block:: yaml
-        parameters:
+
-          archive_command:
+                parameters:
-          - type: String
+                  archive_command:
-            version_from: 90300
+                  - type: String
-            version_till: null
+                    version_from: 90300
-          archive_mode:
+                    version_till: null
-          - type: Bool
+                  archive_mode:
-            version_from: 90300
+                  - type: Bool
-            version_till: 90500
+                    version_from: 90300
-          - type: EnumBool
+                    version_till: 90500
-            version_from: 90500
+                  - type: EnumBool
-            version_till: null
+                    version_from: 90500
-            possible_values:
+                    version_till: null
-            - always
+                    possible_values:
-          archive_timeout:
+                    - always
-          - type: Integer
+                  archive_timeout:
-            version_from: 90300
+                  - type: Integer
-            version_till: null
+                    version_from: 90300
-            min_val: 0
+                    version_till: null
-            max_val: 1073741823
+                    min_val: 0
-            unit: s
+                    max_val: 1073741823
-          autovacuum_vacuum_cost_delay:
+                    unit: s
-          - type: Integer
+                  autovacuum_vacuum_cost_delay:
-            version_from: 90300
+                  - type: Integer
-            version_till: 120000
+                    version_from: 90300
-            min_val: -1
+                    version_till: 120000
-            max_val: 100
+                    min_val: -1
-            unit: ms
+                    max_val: 100
-          - type: Real
+                    unit: ms
-            version_from: 120000
+                  - type: Real
-            version_till: null
+                    version_from: 120000
-            min_val: -1
+                    version_till: null
-            max_val: 100
+                    min_val: -1
-            unit: ms
+                    max_val: 100
-          client_min_messages:
+                    unit: ms
-          - type: Enum
+                  client_min_messages:
-            version_from: 90300
+                  - type: Enum
-            version_till: null
+                    version_from: 90300
-            possible_values:
+                    version_till: null
-            - debug5
+                    possible_values:
-            - debug4
+                    - debug5
-            - debug3
+                    - debug4
-            - debug2
+                    - debug3
-            - debug1
+                    - debug2
-            - log
+                    - debug1
-            - notice
+                    - log
-            - warning
+                    - notice
-            - error
+                    - warning
-        recovery_parameters:
+                    - error
-          archive_cleanup_command:
+                recovery_parameters:
-          - type: String
+                  archive_cleanup_command:
-            version_from: 90300
+                  - type: String
-            version_till: null
+                    version_from: 90300
-        ```
+                    version_till: null
+
     """
     conf_dir = os.path.join(
         os.path.dirname(os.path.abspath(__file__)),
@@ -432,13 +438,15 @@ def _transform_parameter_value(validators: MutableMapping[str, Tuple[_Transforma
     :param value: value of the Postgres GUC.
     :param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres
         GUC. Used for a couple purposes:
+
         * Disallow writing GUCs to ``postgresql.conf`` (or ``recovery.conf``) that does not exist in Postgres *version*;
         * Avoid ignoring GUC *name* if it does not have a validator in *validators*, but is a valid GUC in Postgres
-            *version*.
+          *version*.
 
     :returns: the return value may be one among:
-        * *value* transformed to the expected format for GUC *name* in Postgres *version*, if *name* is present in
+
-            *available_gucs* and has a validator in *validators* for the corresponding Postgres *version*; or
+        * *value* transformed to the expected format for GUC *name* in Postgres *version*, if *name* is present
+          in *available_gucs* and has a validator in *validators* for the corresponding Postgres *version*; or
         * The own *value* if *name* is present in *available_gucs* but not in *validators*; or
         * ``None`` if *name* is not present in *available_gucs*.
     """
@@ -462,11 +470,13 @@ def transform_postgresql_parameter_value(version: int, name: str, value: Any,
     :param value: value of the Postgres GUC.
     :param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres
         GUC. Used for a couple purposes:
-        * Disallow writing GUCs to ``postgresql.conf`` that does not exist in Postgres *version*;
-        * Avoid ignoring GUC *name* if it does not have a validator in ``parameters``, but is a valid GUC in Postgres
-            *version*.
 
-    :returns: The return value may be one among
+            * Disallow writing GUCs to ``postgresql.conf`` that does not exist in Postgres *version*;
+            * Avoid ignoring GUC *name* if it does not have a validator in ``parameters``, but is a valid GUC in
+                Postgres *version*.
+
+    :returns: The return value may be one among:
+
         * The original *value* if *name* seems to be an extension GUC (contains a period '.'); or
         * ``None`` if **name** is a recovery GUC; or
         * *value* transformed to the expected format for GUC *name* in Postgres *version* using validators defined in
@@ -490,10 +500,11 @@ def transform_recovery_parameter_value(version: int, name: str, value: Any,
     :param value: value of the Postgres recovery GUC.
     :param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres
         GUC. Used for a couple purposes:
-        * Disallow writing GUCs to ``recovery.conf`` (or ``postgresql.conf`` depending on *version*), that does not
+
-        exist in Postgres *version*;
+            * Disallow writing GUCs to ``recovery.conf`` (or ``postgresql.conf`` depending on *version*), that does not
-        * Avoid ignoring recovery GUC *name* if it does not have a validator in ``recovery_parameters``, but is a valid
+                exist in Postgres *version*;
-            GUC in Postgres *version*.
+            * Avoid ignoring recovery GUC *name* if it does not have a validator in ``recovery_parameters``, but is a
+                valid GUC in Postgres *version*.
 
     :returns: *value* transformed to the expected format for recovery GUC *name* in Postgres *version* using validators
         defined in ``recovery_parameters``. It can also return ``None``. See :func:`_transform_parameter_value`.

patroni/psycopg.py

@@ -1,7 +1,8 @@
-"""Abstraction layer for ``psycopg`` module.
+"""Abstraction layer for :mod:`psycopg` module.
 
-This module is able to handle both ``pyscopg2`` and ``psycopg3``, and it exposes a common interface for both.
+This module is able to handle both :mod:`pyscopg2` and :mod:`psycopg`, and it exposes a common interface for both.
-``psycopg2`` takes precedence. ``psycopg3`` will only be used if ``psycopg2`` is either absent or older than ``2.5.4``.
+:mod:`psycopg2` takes precedence. :mod:`psycopg` will only be used if :mod:`psycopg2` is either absent or older than
+``2.5.4``.
 """
 from typing import Any, Optional, TYPE_CHECKING, Union
 if TYPE_CHECKING:  # pragma: no cover
@@ -28,7 +29,7 @@ try:
         """Quote *value* as a SQL literal.
 
         .. note::
-            *value* is quoted through ``psycopg`` adapters.
+            *value* is quoted through :mod:`psycopg2` adapters.
 
         :param value: value to be quoted.
         :param conn: if a connection is given then :func:`quote_literal` checks if any special handling based on server
@@ -44,14 +45,14 @@ except ImportError:
     from psycopg import connect as __connect, sql, Error, DatabaseError, OperationalError, ProgrammingError
 
     def _connect(dsn: Optional[str] = None, **kwargs: Any) -> 'Connection[Any]':
-        """Call ``psycopg.connect`` with ``dsn`` and ``**kwargs``.
+        """Call :func:`psycopg.connect` with *dsn* and ``**kwargs``.
 
         .. note::
             Will create ``server_version`` attribute in the returning connection, so it keeps compatibility with the
-            object that would be returned by ``psycopg2.connect``.
+            object that would be returned by :func:`psycopg2.connect`.
 
-        :param dsn: DSN to call ``psycopg.connect`` with.
+        :param dsn: DSN to call :func:`psycopg.connect` with.
-        :param kwargs: keyword arguments to call ``psycopg.connect`` with.
+        :param kwargs: keyword arguments to call :func:`psycopg.connect` with.
 
         :returns: a connection to the database.
         """
@@ -89,11 +90,11 @@ def connect(*args: Any, **kwargs: Any) -> Union['connection', 'Connection[Any]']
         It also enforces ``search_path=pg_catalog`` for non-replication connections to mitigate security issues as
         Patroni relies on superuser connections.
 
-    :param args: positional arguments to call ``connect`` function from ``psycopg`` module.
+    :param args: positional arguments to call :func:`~psycopg.connect` function from :mod:`psycopg` module.
-    :param kwargs: keyword arguments to call ``connect`` function from ``psycopg`` module.
+    :param kwargs: keyword arguments to call :func:`~psycopg.connect` function from :mod:`psycopg` module.
 
-    :returns: a connection to the database. Can be either a :class:`psycopg.Connection` if using ``psycopg3``, or a
+    :returns: a connection to the database. Can be either a :class:`psycopg.Connection` if using :mod:`psycopg`, or a
-        :class:`psycopg2.extensions.connection` if using ``psycopg2``.
+        :class:`psycopg2.extensions.connection` if using :mod:`psycopg2`.
     """
     if kwargs and 'replication' not in kwargs and kwargs.get('fallback_application_name') != 'Patroni ctl':
         options = [kwargs['options']] if 'options' in kwargs else []
@@ -109,7 +110,7 @@ def quote_ident(value: Any, conn: Optional[Union['cursor', 'connection', 'Connec
 
     :param value: value to be quoted.
     :param conn: connection to evaluate the returning string into. Can be either a :class:`psycopg.Connection` if
-        using ``psycopg3``, or a :class:`psycopg2.extensions.connection` if using ``psycopg2``.
+        using :mod:`psycopg`, or a :class:`psycopg2.extensions.connection` if using :mod:`psycopg2`.
 
     :returns: *value* quoted as a SQL identifier.
     """

patroni/request.py

@@ -11,6 +11,19 @@ from .dcs import Member
 from .utils import USER_AGENT
 
 
+class HTTPSConnectionPool(urllib3.HTTPSConnectionPool):
+
+    def _validate_conn(self, *args: Any, **kwargs: Any) -> None:
+        """Override parent method to silence warnings about requests without certificate verification enabled."""
+
+
+class PatroniPoolManager(urllib3.PoolManager):
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super(PatroniPoolManager, self).__init__(*args, **kwargs)
+        self.pool_classes_by_scheme = {'http': urllib3.HTTPConnectionPool, 'https': HTTPSConnectionPool}
+
+
 class PatroniRequest(object):
     """Wrapper for performing requests to Patroni's REST API.
 
@@ -21,29 +34,38 @@ class PatroniRequest(object):
         """Create a new :class:`PatroniRequest` instance with given *config*.
 
         :param config: Patroni YAML configuration.
-        :param insecure: how to deal with SSL certs verification
+        :param insecure: how to deal with SSL certs verification:
+
             * If ``True`` it will perform REST API requests without verifying SSL certs; or
             * If ``False`` it will perform REST API requests and verify SSL certs; or
-            * If ``None`` it will behave according to the value of ``ctl -> insecure`` configuration; or
+            * If ``None`` it will behave according to the value of ``ctl.insecure`` configuration; or
             * If none of the above applies, then it falls back to ``False``.
         """
         self._insecure = insecure
-        self._pool = urllib3.PoolManager(num_pools=10, maxsize=10)
+        self._pool = PatroniPoolManager(num_pools=10, maxsize=10)
         self.reload_config(config)
 
     @staticmethod
-    def _get_cfg_value(config: Union[Config, Dict[str, Any]], name: str) -> Union[Any, None]:
+    def _get_ctl_value(config: Union[Config, Dict[str, Any]], name: str, default: Any = None) -> Optional[Any]:
-        """Get value of *name* setting in *config*.
+        """Get value of *name* setting from the ``ctl`` section of the *config*.
-
-        .. note::
-            *name* key will be searched only under ``ctl`` and ``restapi`` sections, in that order.
 
         :param config: Patroni YAML configuration.
         :param name: name of the setting value to be retrieved.
 
-        :returns: value of ``ctl -> *name*`` or ``restapi -> *name*``, if either is present, ``None`` otherwise.
+        :returns: value of ``ctl.*name*`` if present, ``None`` otherwise.
         """
-        return config.get('ctl', {}).get(name) or config.get('restapi', {}).get(name)
+        return config.get('ctl', {}).get(name, default)
+
+    @staticmethod
+    def _get_restapi_value(config: Union[Config, Dict[str, Any]], name: str) -> Optional[Any]:
+        """Get value of *name* setting from the ``restapi`` section of the *config*.
+
+        :param config: Patroni YAML configuration.
+        :param name: name of the setting value to be retrieved.
+
+        :returns: value of ``restapi -> *name*`` if present, ``None`` otherwise.
+        """
+        return config.get('restapi', {}).get(name)
 
     def _apply_pool_param(self, param: str, value: Any) -> None:
         """Configure *param* as *value* in the request manager.
@@ -62,15 +84,15 @@ class PatroniRequest(object):
 
         :param config: Patroni YAML configuration.
         :param name: prefix of the Patroni SSL related setting name. Currently, supports these:
+
             * ``cert``: gets translated to ``certfile``
             * ``key``: gets translated to ``keyfile``
 
-            Will attempt to fetch the requested key first from ``ctl`` section, and fall back to ``restapi`` section
+            Will attempt to fetch the requested key first from ``ctl`` section.
-            if the former is missing.
 
-        :returns: value of ``ctl -> *name*file`` or ``restapi -> *name*file`` if either is present, ``None`` otherwise.
+        :returns: value of ``ctl.*name*file`` if present, ``None`` otherwise.
         """
-        value = self._get_cfg_value(config, name + 'file')
+        value = self._get_ctl_value(config, name + 'file')
         self._apply_pool_param(name + '_file', value)
         return value
 
@@ -79,37 +101,39 @@ class PatroniRequest(object):
 
         Configure these HTTP headers for requests:
 
-        * ``authorization``: based on Patroni' REST API authentication config;
+            * ``authorization``: based on Patroni' CTL or REST API authentication config;
-        * ``user-agent``: based on `patroni.utils.USER_AGENT`.
+            * ``user-agent``: based on ``patroni.utils.USER_AGENT``.
 
         Also configure SSL related settings for requests:
 
-        * ``ca_certs`` is configured if ``ctl -> cacert`` or ``restapi -> cafile`` is available;
+            * ``ca_certs`` is configured if ``ctl.cacert`` or ``restapi.cafile`` is available;
-        * ``cert``, ``key`` and ``key_password`` are configured if ``ctl -> certile`` or ``restapi -> certfile`` is
+            * ``cert``, ``key`` and ``key_password`` are configured if ``ctl.certfile`` is available.
-            available.
 
         :param config: Patroni YAML configuration.
         """
-        # ``restapi -> auth`` is equivalent to ``restapi -> authentication -> username`` + ``:`` +
+        # ``ctl -> auth`` is equivalent to ``ctl -> authentication -> username`` + ``:`` +
-        # ``restapi -> authentication -> password``
+        # ``ctl -> authentication -> password``. And the same for ``restapi -> auth``
-        self._pool.headers = urllib3.make_headers(basic_auth=self._get_cfg_value(config, 'auth'), user_agent=USER_AGENT)
+        basic_auth = self._get_ctl_value(config, 'auth') or self._get_restapi_value(config, 'auth')
+        self._pool.headers = urllib3.make_headers(basic_auth=basic_auth, user_agent=USER_AGENT)
+        self._pool.connection_pool_kw['cert_reqs'] = 'CERT_REQUIRED'
+
+        insecure = self._insecure if isinstance(self._insecure, bool)\
+            else self._get_ctl_value(config, 'insecure', False)
 
-        insecure = self._insecure if isinstance(self._insecure, bool) else config.get('ctl', {}).get('insecure', False)
         if self._apply_ssl_file_param(config, 'cert'):
-            #  With client certificate the cert_reqs must be set to CERT_REQUIRED even if insecure option is used
+            if insecure:  # The assert_hostname = False helps to silence warnings
-            self._pool.connection_pool_kw['cert_reqs'] = 'CERT_REQUIRED'
+                self._pool.connection_pool_kw['assert_hostname'] = False
-            #  The assert_hostname = False helps to silence warnings
-            self._pool.connection_pool_kw['assert_hostname'] = False if insecure else None
 
             self._apply_ssl_file_param(config, 'key')
-
+            password = self._get_ctl_value(config, 'keyfile_password')
-            password = self._get_cfg_value(config, 'keyfile_password')
             self._apply_pool_param('key_password', password)
         else:
-            self._pool.connection_pool_kw['cert_reqs'] = 'CERT_NONE' if insecure else 'CERT_REQUIRED'
+            if insecure:  # Disable server certificate validation if requested
+                self._pool.connection_pool_kw['cert_reqs'] = 'CERT_NONE'
+            self._pool.connection_pool_kw.pop('assert_hostname', None)
             self._pool.connection_pool_kw.pop('key_file', None)
 
-        cacert = config.get('ctl', {}).get('cacert') or config.get('restapi', {}).get('cafile')
+        cacert = self._get_ctl_value(config, 'cacert') or self._get_restapi_value(config, 'cafile')
         self._apply_pool_param('ca_certs', cacert)
 
     def request(self, method: str, url: str, body: Optional[Any] = None,

patroni/tags.py

@@ -0,0 +1,64 @@
+"""Tags handling."""
+import abc
+
+from typing import Any, Dict, Optional
+
+
+class Tags(abc.ABC):
+    """An abstract class that encapsulates all the ``tags`` logic.
+
+    Child classes that want to use provided facilities must implement ``tags`` abstract property.
+    """
+
+    @staticmethod
+    def _filter_tags(tags: Dict[str, Any]) -> Dict[str, Any]:
+        """Get tags configured for this node, if any.
+
+        Handle both predefined Patroni tags and custom defined tags.
+
+        .. note::
+            A custom tag is any tag added to the configuration ``tags`` section that is not one of ``clonefrom``,
+            ``nofailover``, ``noloadbalance`` or ``nosync``.
+
+            For the Patroni predefined tags, the returning object will only contain them if they are enabled as they
+            all are boolean values that default to disabled.
+
+        :returns: a dictionary of tags set for this node. The key is the tag name, and the value is the corresponding
+            tag value.
+        """
+        return {tag: value for tag, value in tags.items()
+                if tag not in ('clonefrom', 'nofailover', 'noloadbalance', 'nosync') or value}
+
+    @property
+    @abc.abstractmethod
+    def tags(self) -> Dict[str, Any]:
+        """Configured tags.
+
+        Must be implemented in a child class.
+        """
+        raise NotImplementedError  # pragma: no cover
+
+    @property
+    def clonefrom(self) -> bool:
+        """``True`` if ``clonefrom`` tag is ``True``, else ``False``."""
+        return self.tags.get('clonefrom', False)
+
+    @property
+    def nofailover(self) -> bool:
+        """``True`` if ``nofailover`` is ``True``, else ``False``."""
+        return bool(self.tags.get('nofailover', False))
+
+    @property
+    def noloadbalance(self) -> bool:
+        """``True`` if ``noloadbalance`` is ``True``, else ``False``."""
+        return bool(self.tags.get('noloadbalance', False))
+
+    @property
+    def nosync(self) -> bool:
+        """``True`` if ``nosync`` is ``True``, else ``False``."""
+        return bool(self.tags.get('nosync', False))
+
+    @property
+    def replicatefrom(self) -> Optional[str]:
+        """Value of ``replicatefrom`` tag, if any."""
+        return self.tags.get('replicatefrom')

patroni/utils.py

@@ -9,6 +9,8 @@
 :var DBL_RE: regular expression to match double precision numbers, signed or unsigned. Matches scientific notation too.
 :var WHITESPACE_RE: regular expression to match whitespace characters
 """
+import datetime
+import dateutil.parser
 import errno
 import logging
 import os
@@ -16,9 +18,11 @@ import platform
 import random
 import re
 import socket
+import subprocess
 import sys
 import tempfile
 import time
+from enum import Enum
 from shlex import split
 
 from typing import Any, Callable, Dict, Iterator, List, Optional, Union, Tuple, Type, TYPE_CHECKING
@@ -129,8 +133,8 @@ def parse_bool(value: Any) -> Union[bool, None]:
     .. note::
 
         The parsing is case-insensitive, and takes into consideration these values:
-        * ``on``, ``true``, ``yes``, and ``1`` as ``True``.
+            * ``on``, ``true``, ``yes``, and ``1`` as ``True``.
-        * ``off``, ``false``, ``no``, and ``0`` as ``False``.
+            * ``off``, ``false``, ``no``, and ``0`` as ``False``.
 
     :param value: value to be parsed to :class:`bool`.
 
@@ -245,14 +249,16 @@ def convert_to_base_unit(value: Union[int, float], unit: str, base_unit: Optiona
     """Convert *value* as a *unit* of compute information or time to *base_unit*.
 
     :param value: value to be converted to the base unit.
-    :param unit: unit of *value*. Accepts these units (case sensitive)
+    :param unit: unit of *value*. Accepts these units (case sensitive):
-        * For space: ``B``, ``kB``, ``MB``, ``GB``, or ``TB``;
+
-        * For time: ``d``, ``h``, ``min``, ``s``, ``ms``, or ``us``.
+            * For space: ``B``, ``kB``, ``MB``, ``GB``, or ``TB``;
+            * For time: ``d``, ``h``, ``min``, ``s``, ``ms``, or ``us``.
 
     :param base_unit: target unit in the conversion. May contain the target unit with an associated value, e.g
-        ``512MB``. Accepts these units (case sensitive)
+        ``512MB``. Accepts these units (case sensitive):
-        * For space: ``B``, ``kB``, or ``MB``;
+
-        * For time: ``ms``, ``s``, or ``min``.
+            * For space: ``B``, ``kB``, or ``MB``;
+            * For time: ``ms``, ``s``, or ``min``.
 
     :returns: *value* in *unit* converted to *base_unit*. Returns ``None`` if *unit* or *base_unit* is invalid.
 
@@ -402,7 +408,8 @@ def compare_values(vartype: str, unit: Optional[str], old_value: Any, new_value:
     """Check if *old_value* and *new_value* are equivalent after parsing them as *vartype*.
 
     :param vartpe: the target type to parse *old_value* and *new_value* before comparing them. Accepts any among of the
-        following (case sensitive)
+        following (case sensitive):
+
         * ``bool``: parse values using :func:`parse_bool`; or
         * ``integer``: parse values using :func:`parse_int`; or
         * ``real``: parse values using :func:`parse_real`; or
@@ -459,7 +466,7 @@ def compare_values(vartype: str, unit: Optional[str], old_value: Any, new_value:
 
 
 def _sleep(interval: Union[int, float]) -> None:
-    """Wrap :func:`time.sleep`.
+    """Wrap :func:`~time.sleep`.
 
     :param interval: Delay execution for a given number of seconds. The argument may be a floating point number for
         subsecond precision.
@@ -467,6 +474,18 @@ def _sleep(interval: Union[int, float]) -> None:
     time.sleep(interval)
 
 
+def read_stripped(file_path: str) -> Iterator[str]:
+    """Iterate over stripped lines in the given file.
+
+    :param file_path: path to the file to read from
+
+    :yields: each line from the given file stripped
+    """
+    with open(file_path) as f:
+        for line in f:
+            yield line.strip()
+
+
 class RetryFailedError(PatroniException):
     """Maximum number of attempts exhausted in retry operation."""
 
@@ -536,6 +555,7 @@ class Retry(object):
         """Set next cycle delay.
 
         It will be the minimum value between:
+
             * current delay with ``backoff``; or
             * ``max_delay``.
         """
@@ -549,10 +569,14 @@ class Retry(object):
     def ensure_deadline(self, timeout: float, raise_ex: Optional[Exception] = None) -> bool:
         """Calculates, sets, and checks the remaining deadline time.
 
-        :param timeout: if the *deadline* is smaller than the provided *timeout* value raise *raise_ex* exception
+        :param timeout: if the *deadline* is smaller than the provided *timeout* value raise *raise_ex* exception.
-        :param raise_ex: the exception object that will be raised if the *deadline* is smaller than provided *timeout*
+        :param raise_ex: the exception object that will be raised if the *deadline* is smaller than provided *timeout*.
-        :returns: `False` if *deadline* is smaller than a provided *timeout* and *raise_ex* isn't set. Otherwise `True`
+
-        :raises Exception: if calculated deadline is smaller than provided *timeout*
+        :returns: ``False`` if *deadline* is smaller than a provided *timeout* and *raise_ex* isn't set. Otherwise
+            ``True``.
+
+        :raises:
+            :class:`Exception`: *raise_ex* if calculated deadline is smaller than provided *timeout*.
         """
         self.deadline = self.stoptime - time.time()
         if self.deadline < timeout:
@@ -565,9 +589,10 @@ class Retry(object):
         """Call a function *func* with arguments ``*args`` and ``*kwargs`` in a loop.
 
         *func* will be called until one of the following conditions is met:
-        * It completes without throwing one of the configured ``retry_exceptions``; or
+
-        * ``max_retries`` is exceeded.; or
+            * It completes without throwing one of the configured ``retry_exceptions``; or
-        * ``deadline`` is exceeded.
+            * ``max_retries`` is exceeded.; or
+            * ``deadline`` is exceeded.
 
         .. note::
             * It will set loop stop time based on ``deadline`` attribute.
@@ -576,9 +601,10 @@ class Retry(object):
         :param func: function to call.
         :param args: positional arguments to call *func* with.
         :params kwargs: keyword arguments to call *func* with.
-        :raises :class:`RetryFailedError`
+        :raises:
-            * If ``max_tries`` is exceeded; or
+            :class:`RetryFailedError`:
-            * If ``deadline`` is exceeded.
+                * If ``max_tries`` is exceeded; or
+                * If ``deadline`` is exceeded.
         """
         self.reset()
 
@@ -613,7 +639,8 @@ def polling_loop(timeout: Union[int, float], interval: Union[int, float] = 1) ->
 
     :param timeout: for how long (in seconds) from now it should keep returning values.
     :param interval: for how long to sleep before returning a new value.
-    :rtype: Iterator[:class:`int`] with current iteration counter, starting from ``0``.
+
+    :yields: current iteration counter, starting from ``0``.
     """
     start_time = time.time()
     iteration = 0
@@ -627,14 +654,16 @@ def polling_loop(timeout: Union[int, float], interval: Union[int, float] = 1) ->
 def split_host_port(value: str, default_port: Optional[int]) -> Tuple[str, int]:
     """Extract host(s) and port from *value*.
 
-    :param value: string from where host(s) and port will be extracted. Accepts either of these formats
+    :param value: string from where host(s) and port will be extracted. Accepts either of these formats:
-        * ``host:port``; or
+
-        * ``host1,host2,...,hostn:port``.
+            * ``host:port``; or
+            * ``host1,host2,...,hostn:port``.
 
         Each ``host`` portion of *value* can be either:
-        * A FQDN; or
+
-        * An IPv4 address; or
+            * A FQDN; or
-        * An IPv6 address, with or without square brackets.
+            * An IPv4 address; or
+            * An IPv6 address, with or without square brackets.
 
     :param default_port: if no port can be found in *param*, use *default_port* instead.
 
@@ -669,18 +698,23 @@ def uri(proto: str, netloc: Union[List[str], Tuple[str, Union[int, str]], str],
 
     :param proto: the URI protocol.
     :param netloc: the URI host(s) and port. Can be specified in either way among
+
         * A :class:`list` or :class:`tuple`. The second item should be a port, and the first item should be composed of
             hosts in either of these formats:
+
             * ``host``; or.
             * ``host1,host2,...,hostn``.
+
         * A :class:`str` in either of these formats:
+
             * ``host:port``; or
             * ``host1,host2,...,hostn:port``.
 
         In all cases, each ``host`` portion of *netloc* can be either:
-        * An FQDN; or
+
-        * An IPv4 address; or
+            * An FQDN; or
-        * An IPv6 address, with or without square brackets.
+            * An IPv4 address; or
+            * An IPv6 address, with or without square brackets.
 
     :param path: the URI path.
     :param user: the authenticating user, if any.
@@ -698,10 +732,11 @@ def uri(proto: str, netloc: Union[List[str], Tuple[str, Union[int, str]], str],
 
 
 def iter_response_objects(response: HTTPResponse) -> Iterator[Dict[str, Any]]:
-    """Iterate over the chunks of a :class:`HTTPResponse` and yield each JSON document that is found along the way.
+    """Iterate over the chunks of a :class:`~urllib3.response.HTTPResponse` and yield each JSON document that is found.
 
     :param response: the HTTP response from which JSON documents will be retrieved.
-    :rtype: Iterator[:class:`dict`] with current JSON document.
+
+    :yields: current JSON document.
     """
     prev = ''
     decoder = JSONDecoder()
@@ -730,33 +765,36 @@ def iter_response_objects(response: HTTPResponse) -> Iterator[Dict[str, Any]]:
 def cluster_as_json(cluster: 'Cluster', global_config: Optional['GlobalConfig'] = None) -> Dict[str, Any]:
     """Get a JSON representation of *cluster*.
 
-    :param cluster: the :class:`Cluster` object to be parsed as JSON.
+    :param cluster: the :class:`~patroni.dcs.Cluster` object to be parsed as JSON.
-    :param global_config: optional :class:`GlobalConfig` object to check the cluster state.
+    :param global_config: optional :class:`~patroni.config.GlobalConfig` object to check the cluster state.
                           if not provided will be instantiated from the `Cluster.config`.
 
     :returns: JSON representation of *cluster*.
 
     These are the possible keys in the returning object depending on the available information in *cluster*:
 
-    * ``members``: list of members in the cluster. Each value is a :class:`dict` that may have the following keys:
+        * ``members``: list of members in the cluster. Each value is a :class:`dict` that may have the following keys:
-        * ``name``: the name of the host (unique in the cluster). The ``members`` list is sorted by this key;
+
-        * ``role``: ``leader``, ``standby_leader``, ``sync_standby``, or ``replica``;
+            * ``name``: the name of the host (unique in the cluster). The ``members`` list is sorted by this key;
-        * ``state``: ``stopping``, ``stopped``, ``stop failed``, ``crashed``, ``running``, ``starting``,
+            * ``role``: ``leader``, ``standby_leader``, ``sync_standby``, or ``replica``;
-            ``start failed``, ``restarting``, ``restart failed``, ``initializing new cluster``, ``initdb failed``,
+            * ``state``: ``stopping``, ``stopped``, ``stop failed``, ``crashed``, ``running``, ``starting``,
-            ``running custom bootstrap script``, ``custom bootstrap failed``, or ``creating replica``;
+                ``start failed``, ``restarting``, ``restart failed``, ``initializing new cluster``, ``initdb failed``,
-        * ``api_url``: REST API URL based on ``restapi->connect_address`` configuration;
+                ``running custom bootstrap script``, ``custom bootstrap failed``, or ``creating replica``;
-        * ``host``: PostgreSQL host based on ``postgresql->connect_address``;
+            * ``api_url``: REST API URL based on ``restapi->connect_address`` configuration;
-        * ``port``: PostgreSQL port based on ``postgresql->connect_address``;
+            * ``host``: PostgreSQL host based on ``postgresql->connect_address``;
-        * ``timeline``: PostgreSQL current timeline;
+            * ``port``: PostgreSQL port based on ``postgresql->connect_address``;
-        * ``pending_restart``: ``True`` if PostgreSQL is pending to be restarted;
+            * ``timeline``: PostgreSQL current timeline;
-        * ``scheduled_restart``: scheduled restart timestamp, if any;
+            * ``pending_restart``: ``True`` if PostgreSQL is pending to be restarted;
-        * ``tags``: any tags that were set for this member;
+            * ``scheduled_restart``: scheduled restart timestamp, if any;
-        * ``lag``: replication lag, if applicable;
+            * ``tags``: any tags that were set for this member;
-    * ``pause``: ``True`` if cluster is in maintenance mode;
+            * ``lag``: replication lag, if applicable;
-    * ``scheduled_switchover``: if a switchover has been scheduled, then it contains this entry with these keys:
+
-        * ``at``: timestamp when switchover was scheduled to occur;
+        * ``pause``: ``True`` if cluster is in maintenance mode;
-        * ``from``: name of the member to be demoted;
+        * ``scheduled_switchover``: if a switchover has been scheduled, then it contains this entry with these keys:
-        * ``to``: name of the member to be promoted.
+
+            * ``at``: timestamp when switchover was scheduled to occur;
+            * ``from``: name of the member to be demoted;
+            * ``to``: name of the member to be promoted.
     """
     if not global_config:
         from patroni.config import get_global_config
@@ -801,8 +839,9 @@ def cluster_as_json(cluster: 'Cluster', global_config: Optional['GlobalConfig']
         ret['pause'] = True
     if cluster.failover and cluster.failover.scheduled_at:
         ret['scheduled_switchover'] = {'at': cluster.failover.scheduled_at.isoformat()}
-        if cluster.failover.leader:
+        if TYPE_CHECKING:  # pragma: no cover
-            ret['scheduled_switchover']['from'] = cluster.failover.leader
+            assert cluster.failover.leader
+        ret['scheduled_switchover']['from'] = cluster.failover.leader
         if cluster.failover.candidate:
             ret['scheduled_switchover']['to'] = cluster.failover.candidate
     return ret
@@ -832,15 +871,18 @@ def validate_directory(d: str, msg: str = "{} {}") -> None:
         If the directory does not exist, :func:`validate_directory` will attempt to create it.
 
     :param d: the directory to be checked.
-    :param msg: a message to be thrown when raising :class:`PatroniException`, if any issue is faced. It must contain
+    :param msg: a message to be thrown when raising :class:`~patroni.exceptions.PatroniException`, if any issue is
-        2 placeholders to be used by :func:`format`:
+        faced. It must contain 2 placeholders to be used by :func:`format`:
-        * The first placeholder will be replaced with path *d*;
-        * The second placeholder will be replaced with the error condition.
 
-    :raises :class:`PatroniException`: if any issue is observed while validating *d*. Can be thrown in these situations
+            * The first placeholder will be replaced with path *d*;
-        * *d* did not exist, and :func:`validate_directory` was not able to create it; or
+            * The second placeholder will be replaced with the error condition.
-        * *d* is an existing directory, but Patroni is not able to write to that directory; or
+
-        * *d* is an existing file, not a directory.
+    :raises:
+        :class:`~patroni.exceptions.PatroniException`: if any issue is observed while validating *d*. Can be thrown if:
+
+            * *d* did not exist, and :func:`validate_directory` was not able to create it; or
+            * *d* is an existing directory, but Patroni is not able to write to that directory; or
+            * *d* is an existing file, not a directory.
     """
     if not os.path.exists(d):
         try:
@@ -895,13 +937,22 @@ def keepalive_socket_options(timeout: int, idle: int, cnt: int = 3) -> Iterator[
     :param idle: value for ``TCP_KEEPIDLE``.
     :param cnt: value for ``TCP_KEEPCNT``.
 
-    :rtype: Iterator[Tuple[:class:`int`, :class:`int`, :class:`int`]] of all keepalive related socket options to be
+    :yields: all keepalive related socket options to be set. The first item in the tuple is the protocol, the second
-        set. The first item in the tuple is the protocol, the second item is the option, and the third item is the
+        item is the option, and the third item is the value to be used. The return values depend on the platform:
-        value to be used. The return values depend on the platform:
+
-        * ``Windows``: yield ``SO_KEEPALIVE``;
+            * ``Windows``:
-        * ``Linux``: yield ``SO_KEEPALIVE``, ``TCP_USER_TIMEOUT``, ``TCP_KEEPIDLE`, ``TCP_KEEPINTVL``, and
+                * ``SO_KEEPALIVE``.
-            ``TCP_KEEPCNT``;
+            * ``Linux``:
-        * ``MacOS``: yield ``SO_KEEPALIVE``, ``TCP_KEEPIDLE`, ``TCP_KEEPINTVL``, and ``TCP_KEEPCNT``
+                * ``SO_KEEPALIVE``;
+                * ``TCP_USER_TIMEOUT``;
+                * ``TCP_KEEPIDLE``;
+                * ``TCP_KEEPINTVL``;
+                * ``TCP_KEEPCNT``.
+            * ``MacOS``:
+                * ``SO_KEEPALIVE``;
+                * ``TCP_KEEPIDLE``;
+                * ``TCP_KEEPINTVL``;
+                * ``TCP_KEEPCNT``.
     """
     yield (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
 
@@ -939,7 +990,7 @@ def enable_keepalive(sock: socket.socket, timeout: int, idle: int, cnt: int = 3)
     :param idle: value for ``TCP_KEEPIDLE``.
     :param cnt: value for ``TCP_KEEPCNT``.
 
-    :returns: output of :func:`socket.ioctl` if we are on Windows, nothing otherwise.
+    :returns: output of :func:`~socket.ioctl` if we are on Windows, nothing otherwise.
     """
     SIO_KEEPALIVE_VALS = getattr(socket, 'SIO_KEEPALIVE_VALS', None)
     if SIO_KEEPALIVE_VALS is not None:  # Windows
@@ -953,23 +1004,27 @@ def enable_keepalive(sock: socket.socket, timeout: int, idle: int, cnt: int = 3)
 def unquote(string: str) -> str:
     """Unquote a fully quoted *string*.
 
+    :param string: The string to be checked for quoting.
+
+    :returns: The string with quotes removed, if it is a fully quoted single string, or the original string if quoting
+        is not detected, or unquoting was not possible.
+
     :Examples:
 
         A *string* with quotes will have those quotes removed
+
         >>> unquote('"a quoted string"')
         'a quoted string'
 
         A *string* with multiple quotes will be returned as is
+
         >>> unquote('"a multi" "quoted string"')
         '"a multi" "quoted string"'
 
         So will a *string* with unbalanced quotes
+
         >>> unquote('unbalanced "quoted string')
         'unbalanced "quoted string'
-
-    :param string: The string to be checked for quoting.
-    :returns: The string with quotes removed, if it is a fully quoted single string,
-              or the original string if quoting is not detected, or unquoting was not possible.
     """
     try:
         ret = split(string)
@@ -977,3 +1032,56 @@ def unquote(string: str) -> str:
     except ValueError:
         ret = string
     return ret
+
+
+def get_major_version(bin_dir: Optional[str] = None, bin_name: str = 'postgres') -> str:
+    """Get the major version of PostgreSQL.
+
+    It is based on the output of ``postgres --version``.
+
+    :param bin_dir: path to the PostgreSQL binaries directory. If ``None`` or an empty string, it will use the first
+                    *bin_name* binary that is found by the subprocess in the ``PATH``.
+    :param bin_name: name of the postgres binary to call (``postgres`` by default)
+
+    :returns: the PostgreSQL major version.
+
+    :raises:
+        :exc:`~patroni.exceptions.PatroniException`: if the postgres binary call failed due to :exc:`OSError`.
+
+    :Example:
+
+        * Returns `9.6` for PostgreSQL 9.6.24
+        * Returns `15` for PostgreSQL 15.2
+    """
+    if not bin_dir:
+        binary = bin_name
+    else:
+        binary = os.path.join(bin_dir, bin_name)
+    try:
+        version = subprocess.check_output([binary, '--version']).decode()
+    except OSError as e:
+        raise PatroniException(f'Failed to get postgres version: {e}')
+    version = re.match(r'^[^\s]+ [^\s]+ (\d+)(\.(\d+))?', version)
+    if TYPE_CHECKING:  # pragma: no cover
+        assert version is not None
+    return '.'.join([version.group(1), version.group(3)]) if int(version.group(1)) < 10 else version.group(1)
+
+
+class ParseScheduleErrors(Enum):
+    NO_TIMEZONE = ('Timezone information is mandatory for the scheduled {action}', 400)
+    SCHEDULED_IN_PAST = ('Cannot schedule {action} in the past', 422)
+    PARSING_ERROR = ('Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601', 422)
+
+
+def parse_schedule(schedule: Optional[str]) -> Tuple[Optional[ParseScheduleErrors], Optional[datetime.datetime]]:
+    scheduled_at = None
+    if schedule is not None:
+        try:
+            scheduled_at = dateutil.parser.parse(schedule)
+            if scheduled_at.tzinfo is None:
+                return ParseScheduleErrors.NO_TIMEZONE, scheduled_at
+            elif scheduled_at < datetime.datetime.now(tzutc):
+                return ParseScheduleErrors.SCHEDULED_IN_PAST, scheduled_at
+        except (ValueError, TypeError):
+            return ParseScheduleErrors.PARSING_ERROR, scheduled_at
+    return None, scheduled_at

patroni/validator.py

@@ -3,25 +3,26 @@
 
 This module contains facilities for validating configuration of Patroni processes.
 
-:var schema: configuration schema of the daemon launched by `patroni` command.
+:var schema: configuration schema of the daemon launched by ``patroni`` command.
 """
 import os
-import re
 import shutil
 import socket
-import subprocess
 
-from typing import Any, Dict, Union, Iterator, List, Optional as OptionalType, TYPE_CHECKING
+from typing import Any, Dict, Union, Iterator, List, Optional as OptionalType, Tuple
+
+from .collections import CaseInsensitiveSet
 
-from .utils import parse_int, split_host_port, data_directory_is_empty
 from .dcs import dcs_modules
 from .exceptions import ConfigParseError
+from .utils import parse_int, split_host_port, data_directory_is_empty, get_major_version
 
 
 def data_directory_empty(data_dir: str) -> bool:
     """Check if PostgreSQL data directory is empty.
 
     :param data_dir: path to the PostgreSQL data directory to be checked.
+
     :returns: ``True`` if the data directory is empty.
     """
     if os.path.isfile(os.path.join(data_dir, "global", "pg_control")):
@@ -32,12 +33,14 @@ def data_directory_empty(data_dir: str) -> bool:
 def validate_connect_address(address: str) -> bool:
     """Check if options related to connection address were properly configured.
 
-    :param address: address to be validated in the format
+    :param address: address to be validated in the format ``host:ip``.
-        ``host:ip``.
+
     :returns: ``True`` if the address is valid.
-    :raises :class:`patroni.exceptions.ConfigParseError`:
+
-        * If the address is not in the expected format; or
+    :raises:
-        * If the host is set to not allowed values (``127.0.0.1``, ``0.0.0.0``, ``*``, ``::1``, or ``localhost``).
+        :class:`~patroni.exceptions.ConfigParseError`:
+            * If the address is not in the expected format; or
+            * If the host is set to not allowed values (``127.0.0.1``, ``0.0.0.0``, ``*``, ``::1``, or ``localhost``).
     """
     try:
         host, _ = split_host_port(address, 1)
@@ -51,20 +54,25 @@ def validate_connect_address(address: str) -> bool:
 def validate_host_port(host_port: str, listen: bool = False, multiple_hosts: bool = False) -> bool:
     """Check if host(s) and port are valid and available for usage.
 
-    :param host_port: the host(s) and port to be validated. It can be in either of these formats
+    :param host_port: the host(s) and port to be validated. It can be in either of these formats:
+
         * ``host:ip``, if *multiple_hosts* is ``False``; or
         * ``host_1,host_2,...,host_n:port``, if *multiple_hosts* is ``True``.
 
     :param listen: if the address is expected to be available for binding. ``False`` means it expects to connect to that
         address, and ``True`` that it expects to bind to that address.
     :param multiple_hosts: if *host_port* can contain multiple hosts.
+
     :returns: ``True`` if the host(s) and port are valid.
-    :raises: :class:`patroni.exceptions.ConfigParserError`:
+
-        * If the *host_port* is not in the expected format; or
+    :raises:
-        * If ``*`` was specified along with more hosts in *host_port*; or
+        :class:`~patroni.exceptions.ConfigParseError`:
-        * If we are expecting to bind to an address that is already in use; or
+            * If the *host_port* is not in the expected format; or
-        * If we are not able to connect to an address that we are expecting to do so; or
+            * If ``*`` was specified along with more hosts in *host_port*; or
-        * If :class:`socket.gaierror` is thrown by socket module when attempting to connect to the given address(es).
+            * If we are expecting to bind to an address that is already in use; or
+            * If we are not able to connect to an address that we are expecting to do so; or
+            * If :class:`~socket.gaierror` is thrown by socket module when attempting to connect to the given
+              address(es).
     """
     try:
         hosts, port = split_host_port(host_port, 1)
@@ -104,6 +112,7 @@ def validate_host_port_list(value: List[str]) -> bool:
     Call :func:`validate_host_port` with each item in *value*.
 
     :param value: list of host(s) and port items to be validated.
+
     :returns: ``True`` if all items are valid.
     """
     assert all([validate_host_port(v) for v in value]), "didn't pass the validation"
@@ -116,6 +125,7 @@ def comma_separated_host_port(string: str) -> bool:
     Call :func:`validate_host_port_list` with a list represented by the CSV *string*.
 
     :param string: comma-separated list of host and port items.
+
     :returns: ``True`` if all items in the CSV string are valid.
     """
     return validate_host_port_list([s.strip() for s in string.split(",")])
@@ -127,7 +137,7 @@ def validate_host_port_listen(host_port: str) -> bool:
     Call :func:`validate_host_port` with *listen* set to ``True``.
 
     :param host_port: the host and port to be validated. Must be in the format
-        `host:ip`.
+        ``host:ip``.
 
     :returns: ``True`` if the host and port are valid and available for binding.
     """
@@ -140,8 +150,9 @@ def validate_host_port_listen_multiple_hosts(host_port: str) -> bool:
     Call :func:`validate_host_port` with both *listen* and *multiple_hosts* set to ``True``.
 
     :param host_port: the host(s) and port to be validated. It can be in either of these formats
-        * `host:ip`; or
+
-        * `host_1,host_2,...,host_n:port`
+        * ``host:ip``; or
+        * ``host_1,host_2,...,host_n:port``
 
     :returns: ``True`` if the host(s) and port are valid and available for binding.
     """
@@ -152,8 +163,11 @@ def is_ipv4_address(ip: str) -> bool:
     """Check if *ip* is a valid IPv4 address.
 
     :param ip: the IP to be checked.
+
     :returns: ``True`` if the IP is an IPv4 address.
-    :raises :class:`patroni.exceptions.ConfigParserError`: if *ip* is not a valid IPv4 address.
+
+    :raises:
+        :class:`~patroni.exceptions.ConfigParseError`: if *ip* is not a valid IPv4 address.
     """
     try:
         socket.inet_aton(ip)
@@ -166,8 +180,11 @@ def is_ipv6_address(ip: str) -> bool:
     """Check if *ip* is a valid IPv6 address.
 
     :param ip: the IP to be checked.
+
     :returns: ``True`` if the IP is an IPv6 address.
-    :raises :class:`patroni.exceptions.ConfigParserError`: if *ip* is not a valid IPv6 address.
+
+    :raises:
+        :class:`~patroni.exceptions.ConfigParseError`: if *ip* is not a valid IPv6 address.
     """
     try:
         socket.inet_pton(socket.AF_INET6, ip)
@@ -186,31 +203,6 @@ def get_bin_name(bin_name: str) -> str:
     return (schema.data.get('postgresql', {}).get('bin_name', {}) or {}).get(bin_name, bin_name)
 
 
-def get_major_version(bin_dir: OptionalType[str] = None) -> str:
-    """Get the major version of PostgreSQL.
-
-    It is based on the output of ``postgres --version``.
-
-    :param bin_dir: path to PostgreSQL binaries directory. If ``None`` it will use the first ``postgres`` binary that
-        is found by subprocess in the ``PATH``.
-    :returns: the PostgreSQL major version.
-
-    :Example:
-
-        * Returns `9.6` for PostgreSQL 9.6.24
-        * Returns `15` for PostgreSQL 15.2
-    """
-    if not bin_dir:
-        binary = get_bin_name('postgres')
-    else:
-        binary = os.path.join(bin_dir, get_bin_name('postgres'))
-    version = subprocess.check_output([binary, '--version']).decode()
-    version = re.match(r'^[^\s]+ [^\s]+ (\d+)(\.(\d+))?', version)
-    if TYPE_CHECKING:  # pragma: no cover
-        assert version is not None
-    return '.'.join([version.group(1), version.group(3)]) if int(version.group(1)) < 10 else version.group(1)
-
-
 def validate_data_dir(data_dir: str) -> bool:
     """Validate the value of ``postgresql.data_dir`` configuration option.
 
@@ -221,14 +213,17 @@ def validate_data_dir(data_dir: str) -> bool:
         * Point to a non-empty directory that seems to contain a valid PostgreSQL data directory.
 
     :param data_dir: the value of ``postgresql.data_dir`` configuration option.
+
     :returns: ``True`` if the PostgreSQL data directory is valid.
-    :raises :class:`patroni.exceptions.ConfigParserError`:
+
-        * If no *data_dir* was given; or
+    :raises:
-        * If *data_dir* is a file and not a directory; or
+        :class:`~patroni.exceptions.ConfigParseError`:
-        * If *data_dir* is a non-empty directory and:
+            * If no *data_dir* was given; or
-            * ``PG_VERSION`` file is not available in the directory
+            * If *data_dir* is a file and not a directory; or
-            * ``pg_wal``/``pg_xlog`` is not available in the directory
+            * If *data_dir* is a non-empty directory and:
-            * ``PG_VERSION`` content does not match the major version reported by ``postgres --version``
+                * ``PG_VERSION`` file is not available in the directory
+                * ``pg_wal``/``pg_xlog`` is not available in the directory
+                * ``PG_VERSION`` content does not match the major version reported by ``postgres --version``
     """
     if not data_dir:
         raise ConfigParseError("is an empty string")
@@ -245,7 +240,7 @@ def validate_data_dir(data_dir: str) -> bool:
                 raise ConfigParseError("data dir for the cluster is not empty, but doesn't contain"
                                        " \"{}\" directory".format(waldir))
             bin_dir = schema.data.get("postgresql", {}).get("bin_dir", None)
-            major_version = get_major_version(bin_dir)
+            major_version = get_major_version(bin_dir, get_bin_name('postgres'))
             if pgversion != major_version:
                 raise ConfigParseError("data_dir directory postgresql version ({}) doesn't match with "
                                        "'postgres --version' output ({})".format(pgversion, major_version))
@@ -269,11 +264,12 @@ def validate_binary_name(bin_name: str) -> bool:
 
     :returns: ``True`` if the conditions are true
 
-    :raises :class:`patroni.exceptions.ConfigParserError`: if:
+    :raises:
-        * *bin_name* is not set; or
+        :class:`~patroni.exceptions.ConfigParseError` if:
-        * the path join of the ``postgresql.bin_dir`` plus *bin_name* does not exist; or
+            * *bin_name* is not set; or
-        * the path join as above is not executable; or
+            * the path join of the ``postgresql.bin_dir`` plus *bin_name* does not exist; or
-        * the *bin_name* cannot be found in the system PATH
+            * the path join as above is not executable; or
+            * the *bin_name* cannot be found in the system PATH
 
     """
     if not bin_name:
@@ -300,7 +296,7 @@ class Result(object):
 
         .. note::
 
-            ``error`` attribute is only set if ``status`` is failed.
+            ``error`` attribute is only set if *status* is failed.
 
         :param status: if the validation succeeded.
         :param error: error message related to the validation that was performed, if the validation failed.
@@ -337,18 +333,20 @@ class Case(object):
         """Create a :class:`Case` object.
 
         :param schema: the schema for validating a set of attributes that may be available in the configuration.
-            Each key is the configuration that is available in a given scope and that should be validated, and the
+                       Each key is the configuration that is available in a given scope and that should be validated,
-            related value is the validation function or expected type.
+                       and the related value is the validation function or expected type.
 
         :Example:
 
-            Case({
+            .. code-block:: python
-                "host": validate_host_port,
-                "url": str,
-            })
 
-        That will check that ``host`` configuration, if given, is valid based on ``validate_host_port`` function, and
+                Case({
-        will also check that ``url`` configuration, if given, is a ``str`` instance.
+                    "host": validate_host_port,
+                    "url": str,
+                })
+
+        That will check that ``host`` configuration, if given, is valid based on :func:`validate_host_port`, and will
+        also check that ``url`` configuration, if given, is a ``str`` instance.
         """
         self._schema = schema
 
@@ -367,14 +365,16 @@ class Or(object):
 
         :Example:
 
-            Or("host", "hosts"): Case({
+            .. code-block:: python
-                "host": validate_host_port,
-                "hosts": Or(comma_separated_host_port, [validate_host_port]),
-            })
 
-        The outer :class:`Or` is used to define that ``host`` and ``hosts`` are possible options in this scope.
+                Or("host", "hosts"): Case({
-        The inner :class`Or` in the ``hosts`` key value is used to define that ``hosts`` option is valid if either of
+                    "host": validate_host_port,
-            the functions ``comma_separated_host_port`` or ``validate_host_port`` succeed to validate it.
+                    "hosts": Or(comma_separated_host_port, [validate_host_port]),
+                })
+
+            The outer :class:`Or` is used to define that ``host`` and ``hosts`` are possible options in this scope.
+            The inner :class`Or` in the ``hosts`` key value is used to define that ``hosts`` option is valid if either
+            of :func:`comma_separated_host_port` or :func:`validate_host_port` succeed to validate it.
         """
         self.args = args
 
@@ -416,12 +416,12 @@ class Directory(object):
         self.contains_executable = contains_executable
 
     def _check_executables(self, path: OptionalType[str] = None) -> Iterator[Result]:
-        """Check that all executables from contains_executable list exist within the given directory or within PATH.
+        """Check that all executables from contains_executable list exist within the given directory or within ``PATH``.
 
         :param path: optional path to the base directory against which executables will be validated.
-                     If not provided, check within PATH.
+                     If not provided, check within ``PATH``.
-        :rtype: Iterator[:class:`Result`] objects with the error message containing the name of the executable,
+
-                if any check fails.
+        :yields: objects with the error message containing the name of the executable, if any check fails.
         """
         for program in self.contains_executable or []:
             if not shutil.which(program, path=path):
@@ -431,8 +431,9 @@ class Directory(object):
         """Check if the expected paths and executables can be found under *name* directory.
 
         :param name: path to the base directory against which paths and executables will be validated.
-                     Check against PATH if name is not provided.
+                     Check against ``PATH`` if name is not provided.
-        :rtype: Iterator[:class:`Result`] objects with the error message related to the failure, if any check fails.
+
+        :yields: objects with the error message related to the failure, if any check fails.
         """
         if not name:
             yield from self._check_executables()
@@ -480,12 +481,13 @@ class Schema(object):
     be performed against each one of them. The validations will be performed whenever the :class:`Schema` object is
     called, or its :func:`validate` method is called.
 
-    :ivar validator: validator of the configuration schema. Can be any of these
+    :ivar validator: validator of the configuration schema. Can be any of these:
+
         * :class:`str`: defines that a string value is required; or
-        * :class:`type`: any subclass of `type`, defines that a value of the given type is required; or
+        * :class:`type`: any subclass of :class:`type`, defines that a value of the given type is required; or
-        * `callable`: any callable object, defines that validation will follow the code defined in the callable
+        * ``callable``: any callable object, defines that validation will follow the code defined in the callable
-            object. If the callable object contains an ``expected_type`` attribute, then it will check if the
+          object. If the callable object contains an ``expected_type`` attribute, then it will check if the
-            configuration value is of the expected type before calling the code of the callable object; or
+          configuration value is of the expected type before calling the code of the callable object; or
         * :class:`list`: list representing one or more values in the configuration; or
         * :class:`dict`: dictionary representing the YAML configuration tree.
     """
@@ -502,11 +504,12 @@ class Schema(object):
             nodes, when it performs checks of the actual setting values.
 
         :param validator: validator of the configuration schema. Can be any of these:
+
             * :class:`str`: defines that a string value is required; or
             * :class:`type`: any subclass of :class:`type`, defines that a value of the given type is required; or
-            * `callable`: Any callable object, defines that validation will follow the code defined in the callable
+            * ``callable``: Any callable object, defines that validation will follow the code defined in the callable
-                object. If the callable object contains an ``expected_type`` attribute, then it will check if the
+              object. If the callable object contains an ``expected_type`` attribute, then it will check if the
-                configuration value is of the expected type before calling the code of the callable object; or
+              configuration value is of the expected type before calling the code of the callable object; or
             * :class:`list`: list representing it expects to contain one or more values in the configuration; or
             * :class:`dict`: dictionary representing the YAML configuration tree.
 
@@ -514,49 +517,56 @@ class Schema(object):
             to stop.
 
             If *validator* is a :class:`dict`, then you should follow these rules:
+
             * For the keys it can be either:
+
                 * A :class:`str` instance. It will be the name of the configuration option; or
                 * An :class:`Optional` instance. The ``name`` attribute of that object will be the name of the
-                    configuration option, and that class makes this configuration option as optional to the
+                  configuration option, and that class makes this configuration option as optional to the
-                    user, allowing it to not be specified in the YAML; or
+                  user, allowing it to not be specified in the YAML; or
                 * An :class:`Or` instance. The ``args`` attribute of that object will contain a tuple of
                     configuration option names. At least one of them should be specified by the user in the YAML;
+
             * For the values it can be either:
+
                 * A new :class:`dict` instance. It will represent a new level in the YAML configuration tree; or
                 * A :class:`Case` instance. This is required if the key of this value is an :class:`Or` instance,
-                    and the :class:`Case` instance is used to map each of the ``args`` in :class:`Or` to their
+                  and the :class:`Case` instance is used to map each of the ``args`` in :class:`Or` to their
-                    corresponding base validator in :class:`Case`; or
+                  corresponding base validator in :class:`Case`; or
                 * An :class:`Or` instance with one or more base validators; or
                 * A :class:`list` instance with a single item which is the base validator; or
                 * A base validator.
 
         :Example:
 
-            Schema({
+            .. code-block:: python
-                "application_name": str,
-                "bind": {
-                    "host": validate_host,
-                    "port": int,
-                },
-                "aliases": [str],
-                Optional("data_directory"): "/var/lib/myapp",
-                Or("log_to_file", "log_to_db"): Case({
-                    "log_to_file": bool,
-                    "log_to_db": bool,
-                }),
-                "version": Or(int, float),
-            })
 
-        This sample schema defines that your YAML configuration follows these rules:
+                Schema({
-        * It must contain an ``application_name`` entry which value should be a :class:`str` instance;
+                    "application_name": str,
-        * It must contain a ``bind.host`` entry which value should be valid as per function ``validate_host``;
+                    "bind": {
-        * It must contain a ``bind.port`` entry which value should be an :class:`int` instance;
+                        "host": validate_host,
-        * It must contain a ``aliases`` entry which value should be a :class:`list` of :class:`str` instances;
+                        "port": int,
-        * It may optionally contain a ``data_directory`` entry, with a value which should be a string;
+                    },
-        * It must contain at least one of ``log_to_file`` or ``log_to_db``, with a value which should be a
+                    "aliases": [str],
-            :class:`bool` instance;
+                    Optional("data_directory"): "/var/lib/myapp",
-        * It must contain a ``version`` entry which value should be either an :class:`int` or a :class:`float`
+                    Or("log_to_file", "log_to_db"): Case({
-            instance.
+                        "log_to_file": bool,
+                        "log_to_db": bool,
+                    }),
+                    "version": Or(int, float),
+                })
+
+            This sample schema defines that your YAML configuration follows these rules:
+
+                * It must contain an ``application_name`` entry which value should be a :class:`str` instance;
+                * It must contain a ``bind.host`` entry which value should be valid as per function ``validate_host``;
+                * It must contain a ``bind.port`` entry which value should be an :class:`int` instance;
+                * It must contain a ``aliases`` entry which value should be a :class:`list` of :class:`str` instances;
+                * It may optionally contain a ``data_directory`` entry, with a value which should be a string;
+                * It must contain at least one of ``log_to_file`` or ``log_to_db``, with a value which should be a
+                  :class:`bool` instance;
+                * It must contain a ``version`` entry which value should be either an :class:`int` or a :class:`float`
+                  instance.
         """
         self.validator = validator
 
@@ -564,6 +574,7 @@ class Schema(object):
         """Perform validation of data using the rules defined in this schema.
 
         :param data: configuration to be validated against ``validator``.
+
         :returns: list of errors identified while validating the *data*, if any.
         """
         errors: List[str] = []
@@ -578,14 +589,15 @@ class Schema(object):
         It first checks that *data* argument type is compliant with the type of ``validator`` attribute.
 
         Additionally:
-        * If ``validator`` attribute is a callable object, calls it to validate *data* argument. Before doing so, if
+            * If ``validator`` attribute is a callable object, calls it to validate *data* argument. Before doing so, if
-            `validator` contains an ``expected_type`` attribute, check if *data* argument is compliant with that
+                `validator` contains an ``expected_type`` attribute, check if *data* argument is compliant with that
-            expected type.
+                expected type.
-        * If ``validator`` attribute is an iterable object (:class:`dict`, :class:`list`, :class:`Directory` or
+            * If ``validator`` attribute is an iterable object (:class:`dict`, :class:`list`, :class:`Directory` or
-            :class:`Or`), then it iterates over it to validate each of the corresponding entries in *data* argument.
+                :class:`Or`), then it iterates over it to validate each of the corresponding entries in *data* argument.
 
         :param data: configuration to be validated against ``validator``.
-        :rtype: Iterator[:class:`Result`] objects with the error message related to the failure, if any check fails.
+
+        :yields: objects with the error message related to the failure, if any check fails.
         """
         self.data = data
 
@@ -626,7 +638,7 @@ class Schema(object):
 
         Only :class:`dict`, :class:`list`, :class:`Directory` and :class:`Or` objects are considered iterable objects.
 
-        :rtype: Iterator[:class:`Result`] objects with the error message related to the failure, if any check fails.
+        :yields: objects with the error message related to the failure, if any check fails.
         """
         if isinstance(self.validator, dict):
             if not isinstance(self.data, dict):
@@ -654,7 +666,7 @@ class Schema(object):
     def iter_dict(self) -> Iterator[Result]:
         """Iterate over a :class:`dict` based ``validator`` to validate the corresponding entries in ``data``.
 
-        :rtype: Iterator[:class:`Result`] objects with the error message related to the failure, if any check fails.
+        :yields: objects with the error message related to the failure, if any check fails.
         """
         # One key in `validator` attribute (`key` variable) can be mapped to one or more keys in `data` attribute (`d`
         # variable), depending on the `key` type.
@@ -677,12 +689,12 @@ class Schema(object):
                                      path=(d + ("." + v.path if v.path else "")), level=v.level, data=v.data)
 
     def iter_or(self) -> Iterator[Result]:
-        """Perform all validations defined in an `Or` object for a given configuration option.
+        """Perform all validations defined in an :class:`Or` object for a given configuration option.
 
         This method can be only called against leaf nodes in the configuration tree. :class:`Or` objects defined in the
         ``validator`` keys will be handled by :func:`iter_dict` method.
 
-        :rtype: Iterator[:class:`Result`] objects with the error message related to the failure, if any check fails.
+        :yields: objects with the error message related to the failure, if any check fails.
         """
         results: List[Result] = []
         for a in self.validator.args:
@@ -708,7 +720,7 @@ class Schema(object):
 
         :param key: key from the ``validator`` attribute.
 
-        :rtype: Iterator[str], keys that should be used to access corresponding value in the ``data`` attribute.
+        :yields: keys that should be used to access corresponding value in the ``data`` attribute.
         """
         # If the key was defined as a `str` object in `validator` attribute, then it is already the final key to access
         # the `data` dictionary.
@@ -735,12 +747,11 @@ class Schema(object):
 
 
 def _get_type_name(python_type: Any) -> str:
-    """Get a user friendly name for a given Python type.
+    """Get a user-friendly name for a given Python type.
 
     :param python_type: Python type which user friendly name should be taken.
 
-    Returns:
+    :returns: User friendly name of the given Python type.
-        User friendly name of the given Python type.
     """
     types: Dict[Any, str] = {str: 'a string', int: 'an integer', float: 'a number',
                              bool: 'a boolean', list: 'an array', dict: 'a dictionary'}
@@ -761,11 +772,11 @@ def assert_(condition: bool, message: str = "Wrong value") -> None:
 class IntValidator(object):
     """Validate an integer setting.
 
-    :cvar expected_type: the expect Python type for an integer setting (:class:`int`).
+    :cvar expected_type: the expected Python type for an integer setting (:class:`int`).
     :ivar min: minimum allowed value for the setting, if any.
     :ivar max: maximum allowed value for the setting, if any.
-    :ivar base_unit: the base unit to convert the value to before checking if it's within `min` and `max` range.
+    :ivar base_unit: the base unit to convert the value to before checking if it's within *min* and *max* range.
-    :ivar raise_assert: if an ``assert`` call should be performed regarding expected type and valid range.
+    :ivar raise_assert: if an ``assert`` test should be performed regarding expected type and valid range.
     """
 
     expected_type = int
@@ -777,22 +788,24 @@ class IntValidator(object):
         :param min: minimum allowed value for the setting, if any.
         :param max: maximum allowed value for the setting, if any.
         :param base_unit: the base unit to convert the value to before checking if it's within *min* and *max* range.
-        :param raise_assert: if an ``assert`` call should be performed regarding expected type and valid range.
+        :param raise_assert: if an ``assert`` test should be performed regarding expected type and valid range.
         """
         self.min = min
         self.max = max
         self.base_unit = base_unit
         self.raise_assert = raise_assert
 
-    def __call__(self, value: Union[int, str]) -> bool:
+    def __call__(self, value: Any) -> bool:
         """Check if *value* is a valid integer and within the expected range.
 
         .. note::
-            If ``raise_assert`` is ``True`` and *value* is not valid, then an ``AssertionError`` will be triggered.
+            If ``raise_assert`` is ``True`` and *value* is not valid, then an :class:`AssertionError` will be triggered.
+
         :param value: value to be checked against the rules defined for this :class:`IntValidator` instance.
+
         :returns: ``True`` if *value* is valid and within the expected range.
         """
-        value = parse_int(value, self.base_unit) or ""
+        value = parse_int(value, self.base_unit)
         ret = isinstance(value, int)\
             and (self.min is None or value >= self.min)\
             and (self.max is None or value <= self.max)
@@ -802,6 +815,39 @@ class IntValidator(object):
         return ret
 
 
+class EnumValidator(object):
+    """Validate enum setting
+
+    :ivar allowed_values: a ``set`` or ``CaseInsensitiveSet`` object with allowed enum values.
+    :ivar raise_assert: if an ``assert`` call should be performed regarding expected type and valid range.
+    """
+
+    def __init__(self, allowed_values: Tuple[str, ...],
+                 case_sensitive: bool = False, raise_assert: bool = False) -> None:
+        """Create an :class:`EnumValidator` object with given allowed values.
+
+        :param allowed_values: a tuple with allowed enum values
+        :param case_sensitive: set to ``True`` to do case sensitive comparisons
+        :param raise_assert: if an ``assert`` call should be performed regarding expected values.
+        """
+        self.allowed_values = set(allowed_values) if case_sensitive else CaseInsensitiveSet(allowed_values)
+        self.raise_assert = raise_assert
+
+    def __call__(self, value: Any) -> bool:
+        """Check if provided *value* could be found within *allowed_values*.
+
+        .. note::
+            If ``raise_assert`` is ``True`` and *value* is not valid, then an ``AssertionError`` will be triggered.
+        :param value: value to be checked.
+        :returns: ``True`` if *value* could be found within *allowed_values*.
+        """
+        ret = isinstance(value, str) and value in self.allowed_values
+
+        if self.raise_assert:
+            assert_(ret)
+        return ret
+
+
 def validate_watchdog_mode(value: Any) -> None:
     """Validate ``watchdog.mode`` configuration option.
 
@@ -827,15 +873,44 @@ validate_etcd = {
         "srv": str,
         "srv_suffix": str,
         "url": str,
-        "proxy": str})
+        "proxy": str
+    }),
+    Optional("protocol"): str,
+    Optional("username"): str,
+    Optional("password"): str,
+    Optional("cacert"): str,
+    Optional("cert"): str,
+    Optional("key"): str
 }
 
 schema = Schema({
     "name": str,
     "scope": str,
+    Optional("ctl"): {
+        Optional("insecure"): bool,
+        Optional("cacert"): str,
+        Optional("certfile"): str,
+        Optional("keyfile"): str,
+        Optional("keyfile_password"): str
+    },
     "restapi": {
         "listen": validate_host_port_listen,
         "connect_address": validate_connect_address,
+        Optional("authentication"): {
+            "username": str,
+            "password": str
+        },
+        Optional("certfile"): str,
+        Optional("keyfile"): str,
+        Optional("keyfile_password"): str,
+        Optional("cafile"): str,
+        Optional("ciphers"): str,
+        Optional("verify_client"): EnumValidator(("none", "optional", "required"),
+                                                 case_sensitive=True, raise_assert=True),
+        Optional("allowlist"): [str],
+        Optional("allowlist_include_members"): bool,
+        Optional("http_extra_headers"): dict,
+        Optional("https_extra_headers"): dict,
         Optional("request_queue_size"): IntValidator(min=0, max=4096, raise_assert=True)
     },
     Optional("bootstrap"): {
@@ -843,15 +918,64 @@ schema = Schema({
             Optional("ttl"): int,
             Optional("loop_wait"): int,
             Optional("retry_timeout"): int,
-            Optional("maximum_lag_on_failover"): int
+            Optional("maximum_lag_on_failover"): int,
+            Optional("maximum_lag_on_syncnode"): int,
+            Optional("postgresql"): {
+                Optional("parameters"): {
+                    Optional("max_connections"): int,
+                    Optional("max_locks_per_transaction"): int,
+                    Optional("max_prepared_transactions"): int,
+                    Optional("max_replication_slots"): int,
+                    Optional("max_wal_senders"): int,
+                    Optional("max_worker_processes"): int
+                },
+                Optional("use_pg_rewind"): bool,
+                Optional("pg_hba"): [str],
+                Optional("pg_ident"): [str],
+                Optional("pg_ctl_timeout"): int,
+                Optional("use_slots"): bool,
+            },
+            Optional("primary_start_timeout"): int,
+            Optional("primary_stop_timeout"): int,
+            Optional("standby_cluster"): {
+                Or("host", "port", "restore_command"): Case({
+                    "host": str,
+                    "port": int,
+                    "restore_command": str
+                }),
+                Optional("primary_slot_name"): str,
+                Optional("create_replica_methods"): [str],
+                Optional("archive_cleanup_command"): str,
+                Optional("recovery_min_apply_delay"): str
+            },
+            Optional("synchronous_mode"): bool,
+            Optional("synchronous_mode_strict"): bool,
+            Optional("synchronous_node_count"): int
         },
-        Optional("initdb"): [Or(str, dict)]
+        Optional("initdb"): [Or(str, dict)],
+        Optional("method"): str
     },
     Or(*available_dcs): Case({
         "consul": {
             Or("host", "url"): Case({
                 "host": validate_host_port,
-                "url": str})
+                "url": str
+            }),
+            Optional("port"): int,
+            Optional("scheme"): str,
+            Optional("token"): str,
+            Optional("verify"): bool,
+            Optional("cacert"): str,
+            Optional("cert"): str,
+            Optional("key"): str,
+            Optional("dc"): str,
+            Optional("checks"): [str],
+            Optional("register_service"): bool,
+            Optional("service_tags"): [str],
+            Optional("service_check_interval"): str,
+            Optional("service_check_tls_server_name"): str,
+            Optional("consistency"): EnumValidator(('default', 'consistent', 'stale'),
+                                                   case_sensitive=True, raise_assert=True)
         },
         "etcd": validate_etcd,
         "etcd3": validate_etcd,
@@ -869,15 +993,28 @@ schema = Schema({
         },
         "zookeeper": {
             "hosts": Or(comma_separated_host_port, [validate_host_port]),
+            Optional("use_ssl"): bool,
+            Optional("cacert"): str,
+            Optional("cert"): str,
+            Optional("key"): str,
+            Optional("key_password"): str,
+            Optional("verify"): bool,
+            Optional("set_acls"): dict
         },
         "kubernetes": {
             "labels": {},
+            Optional("bypass_api_service"): bool,
             Optional("namespace"): str,
             Optional("scope_label"): str,
             Optional("role_label"): str,
+            Optional("leader_label_value"): str,
+            Optional("follower_label_value"): str,
+            Optional("standby_leader_label_value"): str,
+            Optional("tmp_role_label"): str,
             Optional("use_endpoints"): bool,
             Optional("pod_ip"): Or(is_ipv4_address, is_ipv6_address),
             Optional("ports"): [{"name": str, "port": int}],
+            Optional("cacert"): str,
             Optional("retriable_http_codes"): Or(int, [int]),
         },
     }),
@@ -915,7 +1052,8 @@ schema = Schema({
     },
     Optional("watchdog"): {
         Optional("mode"): validate_watchdog_mode,
-        Optional("device"): str
+        Optional("device"): str,
+        Optional("safety_margin"): int
     },
     Optional("tags"): {
         Optional("nofailover"): bool,

patroni/version.py

@@ -2,4 +2,4 @@
 
 :var __version__: the current Patroni version.
 """
-__version__ = '3.0.4'
+__version__ = '3.1.0'

postgres0.yml

@@ -93,7 +93,7 @@ bootstrap:
   # Additional script to be launched after initial cluster creation (will be passed the connection URL as parameter)
 # post_init: /usr/local/bin/setup_cluster.sh
 
-  # Some additional users users which needs to be created after initializing new cluster
+  # Some additional users which needs to be created after initializing new cluster
   users:
     admin:
       password: admin%

postgres1.yml

@@ -87,7 +87,7 @@ bootstrap:
   # Additional script to be launched after initial cluster creation (will be passed the connection URL as parameter)
 # post_init: /usr/local/bin/setup_cluster.sh
 
-  # Some additional users users which needs to be created after initializing new cluster
+  # Some additional users which needs to be created after initializing new cluster
   users:
     admin:
       password: admin%

postgres2.yml

@@ -84,7 +84,7 @@ bootstrap:
   - encoding: UTF8
   - data-checksums
 
-  # Some additional users users which needs to be created after initializing new cluster
+  # Some additional users which needs to be created after initializing new cluster
   users:
     admin:
       password: admin%
@@ -121,4 +121,4 @@ tags:
     nofailover: false
     noloadbalance: false
     clonefrom: false
-    replicatefrom: postgres1
+#    replicatefrom: postgresql1

requirements.docs.txt

@@ -0,0 +1,4 @@
+sphinx>=4
+sphinx_rtd_theme>1
+sphinxcontrib-apidoc
+sphinx-github-style<1.0.3

tests/__init__.py

@@ -104,9 +104,9 @@ class MockCursor(object):
         elif sql.startswith('SELECT slot_name, slot_type, datname, plugin, catalog_xmin'):
             self.results = [('ls', 'logical', 'a', 'b', 100, 500, b'123456')]
         elif sql.startswith('SELECT slot_name'):
-            self.results = [('blabla', 'physical'), ('foobar', 'physical'), ('ls', 'logical', 'a', 'b', 5, 100, 500)]
+            self.results = [('blabla', 'physical'), ('foobar', 'physical'), ('ls', 'logical', 'b', 'a', 5, 100, 500)]
         elif sql.startswith('WITH slots AS (SELECT slot_name, active'):
-            self.results = [(False, True)] if self.rowcount == 1 else [None]
+            self.results = [(False, True)] if self.rowcount == 1 else []
         elif sql.startswith('SELECT CASE WHEN pg_catalog.pg_is_in_recovery()'):
             self.results = [(1, 2, 1, 0, False, 1, 1, None, None, 'streaming', '',
                              [{"slot_name": "ls", "confirmed_flush_lsn": 12345}],
@@ -114,10 +114,22 @@ class MockCursor(object):
         elif sql.startswith('SELECT pg_catalog.pg_is_in_recovery()'):
             self.results = [(False, 2)]
         elif sql.startswith('SELECT pg_catalog.pg_postmaster_start_time'):
-            replication_info = '[{"application_name":"walreceiver","client_addr":"1.2.3.4",' +\
+            self.results = [(datetime.datetime.now(tzutc),)]
-                               '"state":"streaming","sync_state":"async","sync_priority":0}]'
+        elif sql.startswith('SELECT name, current_setting(name) FROM pg_settings'):
-            now = datetime.datetime.now(tzutc)
+            self.results = [('data_directory', 'data'),
-            self.results = [(now, 0, '', 0, '', False, now, 'streaming', None, replication_info)]
+                            ('hba_file', os.path.join('data', 'pg_hba.conf')),
+                            ('ident_file', os.path.join('data', 'pg_ident.conf')),
+                            ('max_connections', 42),
+                            ('max_locks_per_transaction', 73),
+                            ('max_prepared_transactions', 0),
+                            ('max_replication_slots', 21),
+                            ('max_wal_senders', 37),
+                            ('track_commit_timestamp', 'off'),
+                            ('wal_level', 'replica'),
+                            ('listen_addresses', '6.6.6.6'),
+                            ('port', 1984),
+                            ('archive_command', 'my archive command'),
+                            ('cluster_name', 'my_cluster')]
         elif sql.startswith('SELECT name, setting'):
             self.results = [('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
                             ('wal_block_size', '8192', None, 'integer', 'internal'),
@@ -128,6 +140,8 @@ class MockCursor(object):
                             ('listen_addresses', '*', None, 'string', 'postmaster'),
                             ('autovacuum', 'on', None, 'bool', 'sighup'),
                             ('unix_socket_directories', '/tmp', None, 'string', 'postmaster')]
+        elif sql.startswith('SELECT COUNT(*) FROM pg_catalog.pg_settings'):
+            self.results = [(1,)]
         elif sql.startswith('IDENTIFY_SYSTEM'):
             self.results = [('1', 3, '0/402EEC0', '')]
         elif sql.startswith('TIMELINE_HISTORY '):
@@ -141,6 +155,7 @@ class MockCursor(object):
             self.results = [(1, 0, 'host1', 5432, 'primary'), (2, 1, 'host2', 5432, 'primary')]
         else:
             self.results = [(None, None, None, None, None, None, None, None, None, None)]
+        self.rowcount = len(self.results)
 
     def fetchone(self):
         return self.results[0]
@@ -159,11 +174,20 @@ class MockCursor(object):
         pass
 
 
+class MockConnectionInfo(object):
+
+    def parameter_status(self, param_name):
+        if param_name == 'is_superuser':
+            return 'on'
+        return '0'
+
+
 class MockConnect(object):
 
     server_version = 99999
     autocommit = False
     closed = 0
+    info = MockConnectionInfo()
 
     def cursor(self):
         return MockCursor(self)

tests/test_api.py

@@ -3,8 +3,6 @@ import json
 import unittest
 import socket
 
-import patroni.psycopg as psycopg
-
 from http.server import HTTPServer
 from io import BytesIO as IO
 from mock import Mock, PropertyMock, patch
@@ -13,19 +11,43 @@ from socketserver import ThreadingMixIn
 from patroni.api import RestApiHandler, RestApiServer
 from patroni.config import GlobalConfig
 from patroni.dcs import ClusterConfig, Member
+from patroni.exceptions import PostgresConnectionException
 from patroni.ha import _MemberStatus
-from patroni.utils import tzutc
+from patroni.manual_failover import ManualFailoverPrecheckStatus
+from patroni.psycopg import OperationalError
+from patroni.utils import ParseScheduleErrors, RetryFailedError, tzutc
 
-from . import psycopg_connect, MockCursor
+from . import MockConnect, psycopg_connect
-from .test_ha import get_cluster_initialized_without_leader
+from .test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader
 
 
 future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
 postmaster_start_time = datetime.datetime.now(tzutc)
 
 
-class MockPostgresql(object):
+class MockConnection:
 
+    @staticmethod
+    def get(*args):
+        return psycopg_connect()
+
+    @staticmethod
+    def query(sql, *params):
+        return [(postmaster_start_time, 0, '', 0, '', False, postmaster_start_time, 'streaming', None,
+                 '[{"application_name":"walreceiver","client_addr":"1.2.3.4",'
+                 + '"state":"streaming","sync_state":"async","sync_priority":0}]')]
+
+
+class MockConnectionPool:
+
+    @staticmethod
+    def get(*args):
+        return MockConnection()
+
+
+class MockPostgresql:
+
+    connection_pool = MockConnectionPool()
     name = 'test'
     state = 'running'
     role = 'primary'
@@ -36,14 +58,11 @@ class MockPostgresql(object):
     pending_restart = True
     wal_name = 'wal'
     lsn_name = 'lsn'
+    wal_flush = '_flush'
     POSTMASTER_START_TIME = 'pg_catalog.pg_postmaster_start_time()'
     TL_LSN = 'CASE WHEN pg_catalog.pg_is_in_recovery()'
     citus_handler = Mock()
 
-    @staticmethod
-    def connection():
-        return psycopg_connect()
-
     @staticmethod
     def postmaster_start_time():
         return postmaster_start_time
@@ -100,7 +119,7 @@ class MockHa(object):
 
     @staticmethod
     def fetch_nodes_statuses(members):
-        return [_MemberStatus(None, True, None, 0, 0, None, {}, False)]
+        return [_MemberStatus(None, True, None, 0, {})]
 
     @staticmethod
     def schedule_future_restart(data):
@@ -122,6 +141,9 @@ class MockHa(object):
     def is_paused():
         return True
 
+    def has_members_eligible_to_promote(*args, **kwargs):
+        return True
+
 
 class MockLogger(object):
 
@@ -226,7 +248,7 @@ class TestRestApiHandler(unittest.TestCase):
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /primary'))
         with patch.object(RestApiServer, 'query', Mock(return_value=[('', 1, '', '', '', '', False, None, None, '')])):
             self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
-        with patch.object(GlobalConfig, 'is_standby_cluster', Mock(return_value=True)),\
+        with patch.object(GlobalConfig, 'is_standby_cluster', Mock(return_value=True)), \
                 patch.object(GlobalConfig, 'is_paused', Mock(return_value=True)):
             MockRestApiServer(RestApiHandler, 'GET /standby_leader')
 
@@ -487,9 +509,7 @@ class TestRestApiHandler(unittest.TestCase):
 
     @patch('time.sleep', Mock())
     def test_RestApiServer_query(self):
-        with patch.object(MockCursor, 'execute', Mock(side_effect=psycopg.OperationalError)):
+        with patch.object(MockConnection, 'query', Mock(side_effect=RetryFailedError('bla'))):
-            self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
-        with patch.object(MockPostgresql, 'connection', Mock(side_effect=psycopg.OperationalError)):
             self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
 
     @patch('time.sleep', Mock())
@@ -500,86 +520,185 @@ class TestRestApiHandler(unittest.TestCase):
 
         post = 'POST /switchover HTTP/1.0' + self._authorization + '\nContent-Length: '
 
-        MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}')
+        # Invalid content
+        with patch.object(RestApiHandler, 'write_response') as response_mock:
+            MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}')
+            response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SWITCHOVER_NO_LEADER.value[::-1])
 
+        # Empty content
         request = post + '0\n\n'
         MockRestApiServer(RestApiHandler, request)
 
-        cluster.leader.name = 'postgresql1'
+        # [Switchover without a candidate]
-        MockRestApiServer(RestApiHandler, request)
 
+        cluster.leader.name = 'postgresql1'
         request = post + '25\n\n{"leader": "postgresql1"}'
 
-        with patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
+        # No candidate in pause mode
+        with patch.object(RestApiHandler, 'write_response') as response_mock, \
+             patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
             MockRestApiServer(RestApiHandler, request)
+            response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SWITCHOVER_PAUSE_NO_CANDIDATE.value[::-1])
 
-        for is_synchronous_mode in (True, False):
+        # No healthy nodes to promote in both sync and async mode
-            with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)):
+        for is_synchronous_mode, response in (
+                (True, ManualFailoverPrecheckStatus.NO_SYNC_CANDIDATE.value[0].format(action='switchover')),
+                (False, ManualFailoverPrecheckStatus.ONLY_LEADER.value[0].format(action='switchover'))):
+            with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \
+                 patch.object(RestApiHandler, 'write_response') as response_mock:
                 MockRestApiServer(RestApiHandler, request)
+                response_mock.assert_called_with(412, response)
 
-        cluster.leader.name = 'postgresql2'
+        # [Switchover to the candidate specified]
-        request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
-        MockRestApiServer(RestApiHandler, request)
 
+        # Candidate to promote is the same as the leader specified
+        with patch.object(RestApiHandler, 'write_response') as response_mock:
+            request = post + '53\n\n{"leader": "postgresql2", "candidate": "postgresql2"}'
+            MockRestApiServer(RestApiHandler, request)
+            response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SWITCHOVER_TO_LEADER.value[::-1])
+
+        # Current leader is different from the one specified
+        with patch.object(RestApiHandler, 'write_response') as response_mock:
+            cluster.leader.name = 'postgresql2'
+            request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
+            MockRestApiServer(RestApiHandler, request)
+            response_mock.assert_called_with(
+                ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER.value[1],
+                ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER.value[0].format(leader='postgresql1',
+                                                                               cluster_name='dummy'))
+
+        # Candidate to promote is not a sync standby/a member of the cluster
         cluster.leader.name = 'postgresql1'
         cluster.sync.matches.return_value = False
-        for is_synchronous_mode in (True, False):
+        for is_synchronous_mode, response in (
-            with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)):
+                (True, ManualFailoverPrecheckStatus.CANDIDATE_NOT_SYNC_STANDBY.value[0]),
+                (False, ManualFailoverPrecheckStatus.CANDIDATE_NOT_MEMEBER.value[0].format(candidate="postgresql2",
+                                                                                           cluster_name='dummy'))):
+            with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \
+                 patch.object(RestApiHandler, 'write_response') as response_mock:
                 MockRestApiServer(RestApiHandler, request)
+                response_mock.assert_called_with(412, response)
 
         cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
                            Member(0, 'postgresql2', 30, {'api_url': 'http'})]
-        MockRestApiServer(RestApiHandler, request)
 
-        cluster.failover = None
+        # Cluster has no leader
-        MockRestApiServer(RestApiHandler, request)
+        cluster.leader.name = None
+        with patch.object(RestApiHandler, 'write_response') as response_mock:
+            request = post + '53\n\n{"leader": "postgresql1"}'
+            MockRestApiServer(RestApiHandler, request)
+            response_mock.assert_called_with(
+                ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER.value[1],
+                ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER.value[0].format(leader='leader', cluster_name='dummy'))
 
-        dcs.get_cluster.side_effect = [cluster]
+        cluster.leader.name = 'postgresql1'
-        MockRestApiServer(RestApiHandler, request)
 
-        cluster2 = cluster.copy()
+        # Failover key is empty in DCS
-        cluster2.leader.name = 'postgresql0'
+        with patch.object(RestApiHandler, 'write_response') as response_mock:
-        cluster2.is_unlocked.return_value = False
+            cluster.failover = None
-        dcs.get_cluster.side_effect = [cluster, cluster2]
+            request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
-        MockRestApiServer(RestApiHandler, request)
+            MockRestApiServer(RestApiHandler, request)
+            response_mock.assert_called_with(503, 'Switchover failed')
 
-        cluster2.leader.name = 'postgresql2'
+        # Result polling failed
-        dcs.get_cluster.side_effect = [cluster, cluster2]
+        with patch.object(RestApiHandler, 'write_response') as response_mock:
-        MockRestApiServer(RestApiHandler, request)
+            dcs.get_cluster.side_effect = [cluster]
+            MockRestApiServer(RestApiHandler, request)
+            response_mock.assert_called_with(503, 'Switchover status unknown')
+
+        # Switchover to a node different from the candidate specified
+        with patch.object(RestApiHandler, 'write_response') as response_mock:
+            cluster2 = cluster.copy()
+            cluster2.leader.name = 'postgresql0'
+            cluster2.is_unlocked.return_value = False
+            dcs.get_cluster.side_effect = [cluster, cluster2]
+            MockRestApiServer(RestApiHandler, request)
+            response_mock.assert_called_with(200, 'Switched over to "postgresql0" instead of "postgresql2"')
+
+        # Successful switchover to the candidate
+        with patch.object(RestApiHandler, 'write_response') as response_mock:
+            cluster2.leader.name = 'postgresql2'
+            dcs.get_cluster.side_effect = [cluster, cluster2]
+            MockRestApiServer(RestApiHandler, request)
+            response_mock.assert_called_with(200, 'Successfully switched over to "postgresql2"')
+
+        with patch.object(RestApiHandler, 'write_response') as response_mock:
+            dcs.manual_failover.return_value = False
+            dcs.get_cluster.side_effect = None
+            MockRestApiServer(RestApiHandler, request)
+            response_mock.assert_called_with(503, 'failed to write failover key into DCS')
 
-        dcs.get_cluster.side_effect = None
-        dcs.manual_failover.return_value = False
-        MockRestApiServer(RestApiHandler, request)
         dcs.manual_failover.return_value = True
 
-        with patch.object(MockHa, 'fetch_nodes_statuses', Mock(return_value=[])):
+        # Candidate is not healthy to be promoted
+        with patch.object(MockHa, 'has_members_eligible_to_promote', Mock(return_value=False)), \
+             patch.object(RestApiHandler, 'write_response') as response_mock:
             MockRestApiServer(RestApiHandler, request)
+            response_mock.assert_called_with(
+                ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[1],
+                ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[0].format(action='switchover'))
+
+        # [Scheduled switchover]
 
         # Valid future date
-        request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2",' +\
+        with patch.object(RestApiHandler, 'write_response') as response_mock:
-                         ' "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}'
+            request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2",' + \
-        MockRestApiServer(RestApiHandler, request)
+                             ' "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}'
-        with patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)),\
-                patch.object(MockPatroni, 'dcs') as d:
-            d.manual_failover.return_value = False
             MockRestApiServer(RestApiHandler, request)
+            response_mock.assert_called_with(202, 'Switchover scheduled')
 
-        # Exception: No timezone specified
+        # Scheduled in pause mode
-        request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' +\
+        with patch.object(RestApiHandler, 'write_response') as response_mock, \
-                         ' "scheduled_at": "6016-02-15T18:13:30.568224"}'
+             patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
-        MockRestApiServer(RestApiHandler, request)
+            dcs.manual_failover.return_value = False
+            MockRestApiServer(RestApiHandler, request)
+            response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SCHEDULED_SWITCHOVER_PAUSE.value[::-1])
+
+        # No timezone specified
+        with patch.object(RestApiHandler, 'write_response') as response_mock:
+            request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' + \
+                             ' "scheduled_at": "6016-02-15T18:13:30.568224"}'
+            MockRestApiServer(RestApiHandler, request)
+            response_mock.assert_called_with(
+                ParseScheduleErrors.NO_TIMEZONE.value[1],
+                ParseScheduleErrors.NO_TIMEZONE.value[0].format(action='switchover'))
 
-        # Exception: Scheduled in the past
         request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "'
-        MockRestApiServer(RestApiHandler, request + '1016-02-15T18:13:30.568224+01:00"}')
+
+        # Scheduled in the past
+        with patch.object(RestApiHandler, 'write_response') as response_mock:
+            MockRestApiServer(RestApiHandler, request + '1016-02-15T18:13:30.568224+01:00"}')
+            response_mock.assert_called_with(
+                ParseScheduleErrors.SCHEDULED_IN_PAST.value[1],
+                ParseScheduleErrors.SCHEDULED_IN_PAST.value[0].format(action='switchover'))
 
         # Invalid date
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}'))
+        with patch.object(RestApiHandler, 'write_response') as response_mock:
+            MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}')
+            response_mock.assert_called_with(*ParseScheduleErrors.PARSING_ERROR.value[::-1])
 
-    def test_do_POST_failover(self):
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_POST_failover(self, mock_dcs):
         post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
-        MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}')
+        cluster = mock_dcs.get_cluster.return_value
-        MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}')
+
+        with patch.object(RestApiHandler, 'write_response') as response_mock:
+            MockRestApiServer(RestApiHandler, post + '19\n\n{"leader":"leader"}')
+            response_mock.assert_called_once_with(*ManualFailoverPrecheckStatus.FAILOVER_NO_CANDIDATE.value[::-1])
+
+        with patch.object(RestApiHandler, 'write_response') as response_mock:
+            MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}')
+            response_mock.assert_called_once_with(*ManualFailoverPrecheckStatus.SCHEDULED_FAILOVER.value[::-1])
+
+        # Candidate is not healthy to be promoted
+        cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
+                           Member(0, 'postgresql2', 30, {'api_url': 'http'})]
+        with patch.object(MockHa, 'has_members_eligible_to_promote', Mock(return_value=False)), \
+             patch.object(RestApiHandler, 'write_response') as response_mock:
+            MockRestApiServer(RestApiHandler, post + '27\n\n{"candidate":"postgresql2"}')
+            response_mock.assert_called_with(
+                ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[1],
+                ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[0].format(action='failover'))
 
     @patch.object(MockHa, 'is_leader', Mock(return_value=True))
     def test_do_POST_citus(self):
@@ -661,3 +780,11 @@ class TestRestApiServer(unittest.TestCase):
 
     def test_get_certificate_serial_number(self):
         self.assertIsNone(self.srv.get_certificate_serial_number())
+
+    def test_query(self):
+        with patch.object(MockConnection, 'get', Mock(side_effect=OperationalError)):
+            self.assertRaises(PostgresConnectionException, self.srv.query, 'SELECT 1')
+        with patch.object(MockConnection, 'get', Mock(side_effect=[MockConnect(), OperationalError])), \
+                patch.object(MockConnection, 'query') as mock_query:
+            self.srv.query('SELECT 1')
+            mock_query.assert_called_once_with('SELECT 1')

tests/test_bootstrap.py

@@ -155,9 +155,9 @@ class TestBootstrap(BaseTestPostgresql):
 
         config = {'users': {'replicator': {'password': 'rep-pass', 'options': ['replication']}}}
 
-        with patch.object(Postgresql, 'is_running', Mock(return_value=False)),\
+        with patch.object(Postgresql, 'is_running', Mock(return_value=False)), \
-                patch.object(Postgresql, 'get_major_version', Mock(return_value=140000)),\
+                patch.object(Postgresql, 'get_major_version', Mock(return_value=140000)), \
-                patch('multiprocessing.Process', Mock(side_effect=Exception)),\
+                patch('multiprocessing.Process', Mock(side_effect=Exception)), \
                 patch('multiprocessing.get_context', Mock(side_effect=Exception), create=True):
             self.assertRaises(Exception, self.b.bootstrap, config)
         with open(os.path.join(self.p.data_dir, 'pg_hba.conf')) as f:
@@ -185,12 +185,12 @@ class TestBootstrap(BaseTestPostgresql):
         self.assertFalse(self.b.bootstrap(config))
 
         mock_cancellable_subprocess_call.return_value = 0
-        with patch('multiprocessing.Process', Mock(side_effect=Exception("42"))),\
+        with patch('multiprocessing.Process', Mock(side_effect=Exception("42"))), \
-                patch('multiprocessing.get_context', Mock(side_effect=Exception("42")), create=True),\
+                patch('multiprocessing.get_context', Mock(side_effect=Exception("42")), create=True), \
-                patch('os.path.isfile', Mock(return_value=True)),\
+                patch('os.path.isfile', Mock(return_value=True)), \
-                patch('os.unlink', Mock()),\
+                patch('os.unlink', Mock()), \
-                patch.object(ConfigHandler, 'save_configuration_files', Mock()),\
+                patch.object(ConfigHandler, 'save_configuration_files', Mock()), \
-                patch.object(ConfigHandler, 'restore_configuration_files', Mock()),\
+                patch.object(ConfigHandler, 'restore_configuration_files', Mock()), \
                 patch.object(ConfigHandler, 'write_recovery_conf', Mock()):
             with self.assertRaises(Exception) as e:
                 self.b.bootstrap(config)
@@ -250,7 +250,7 @@ class TestBootstrap(BaseTestPostgresql):
         self.assertFalse(self.b.call_post_bootstrap({'post_init': '/bin/false'}))
 
         mock_cancellable_subprocess_call.return_value = 0
-        self.p.config.superuser.pop('username')
+        self.p.connection_pool._conn_kwargs.pop('user')
         self.assertTrue(self.b.call_post_bootstrap({'post_init': '/bin/false'}))
         mock_cancellable_subprocess_call.assert_called()
         args, kwargs = mock_cancellable_subprocess_call.call_args
@@ -258,7 +258,7 @@ class TestBootstrap(BaseTestPostgresql):
         self.assertEqual(args[0], ['/bin/false', 'dbname=postgres host=127.0.0.2 port=5432'])
 
         mock_cancellable_subprocess_call.reset_mock()
-        self.p.config._local_address.pop('host')
+        self.p.connection_pool._conn_kwargs.pop('host')
         self.assertTrue(self.b.call_post_bootstrap({'post_init': '/bin/false'}))
         mock_cancellable_subprocess_call.assert_called()
         self.assertEqual(mock_cancellable_subprocess_call.call_args[0][0], ['/bin/false', 'dbname=postgres port=5432'])

tests/test_citus.py

@@ -13,7 +13,7 @@ class TestCitus(BaseTestPostgresql):
     def setUp(self):
         super(TestCitus, self).setUp()
         self.c = self.p.citus_handler
-        self.c.set_conn_kwargs({'host': 'localhost', 'dbname': 'postgres'})
+        self.p.connection_pool.conn_kwargs = {'host': 'localhost', 'dbname': 'postgres'}
         self.cluster = get_cluster_initialized_with_leader()
         self.cluster.workers[1] = self.cluster
 
@@ -52,7 +52,7 @@ class TestCitus(BaseTestPostgresql):
                                                'leader': 'leader', 'timeout': 30, 'cooldown': 10})
 
     def test_add_task(self):
-        with patch('patroni.postgresql.citus.logger.error') as mock_logger,\
+        with patch('patroni.postgresql.citus.logger.error') as mock_logger, \
                 patch('patroni.postgresql.citus.urlparse', Mock(side_effect=Exception)):
             self.c.add_task('', 1, None)
             mock_logger.assert_called_once()
@@ -107,7 +107,7 @@ class TestCitus(BaseTestPostgresql):
         self.c.process_tasks()
 
         self.c.add_task('after_promote', 0, 'postgres://host3:5432/postgres')
-        with patch('patroni.postgresql.citus.logger.error') as mock_logger,\
+        with patch('patroni.postgresql.citus.logger.error') as mock_logger, \
                 patch.object(CitusHandler, 'query', Mock(side_effect=Exception)):
             self.c.process_tasks()
             mock_logger.assert_called_once()

tests/test_config.py

@@ -21,7 +21,8 @@ class TestConfig(unittest.TestCase):
         with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
             self.assertFalse(self.config.set_dynamic_configuration({'foo': 'bar'}))
         self.assertTrue(self.config.set_dynamic_configuration({'standby_cluster': {}, 'postgresql': {
-            'parameters': {'cluster_name': 1, 'wal_keep_size': 1, 'track_commit_timestamp': 1, 'wal_level': 1}}}))
+            'parameters': {'cluster_name': 1, 'hot_standby': 1, 'wal_keep_size': 1,
+                           'track_commit_timestamp': 1, 'wal_level': 1}}}))
 
     def test_reload_local_configuration(self):
         os.environ.update({
@@ -84,6 +85,7 @@ class TestConfig(unittest.TestCase):
     @patch('os.path.exists', Mock(return_value=True))
     @patch('os.remove', Mock(side_effect=IOError))
     @patch('os.close', Mock(side_effect=IOError))
+    @patch('os.chmod', Mock())
     @patch('shutil.move', Mock(return_value=None))
     @patch('json.dump', Mock())
     def test_save_cache(self):

tests/test_config_generator.py

@@ -0,0 +1,334 @@
+import os
+import psutil
+import socket
+import unittest
+
+from . import MockConnect, MockCursor, MockConnectionInfo
+from copy import deepcopy
+from mock import MagicMock, Mock, PropertyMock, mock_open, patch
+
+from patroni.__main__ import main as _main
+from patroni.config import Config
+from patroni.config_generator import AbstractConfigGenerator, get_address
+
+from patroni.utils import patch_config
+
+from . import psycopg_connect
+
+
+@patch('patroni.psycopg.connect', psycopg_connect)
+@patch('socket.getaddrinfo', Mock(return_value=[(0, 0, 0, 0, ('1.9.8.4', 1984))]))
+@patch('builtins.open', MagicMock())
+@patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 16.2"))
+@patch('psutil.Process.exe', Mock(return_value='/bin/dir/from/running/postgres'))
+@patch('psutil.Process.__init__', Mock(return_value=None))
+class TestGenerateConfig(unittest.TestCase):
+
+    no_value_msg = '#FIXME'
+    _HOSTNAME = socket.gethostname()
+    _IP = sorted(socket.getaddrinfo(_HOSTNAME, 0, socket.AF_UNSPEC, socket.SOCK_STREAM, 0), key=lambda x: x[0])[0][4][0]
+
+    def setUp(self):
+        self.maxDiff = None
+
+        os.environ['PATRONI_SCOPE'] = 'scope_from_env'
+        os.environ['PATRONI_POSTGRESQL_BIN_DIR'] = '/bin/from/env'
+        os.environ['PATRONI_SUPERUSER_USERNAME'] = 'su_user_from_env'
+        os.environ['PATRONI_SUPERUSER_PASSWORD'] = 'su_pwd_from_env'
+        os.environ['PATRONI_REPLICATION_USERNAME'] = 'repl_user_from_env'
+        os.environ['PATRONI_REPLICATION_PASSWORD'] = 'repl_pwd_from_env'
+        os.environ['PATRONI_REWIND_USERNAME'] = 'rewind_user_from_env'
+        os.environ['PGUSER'] = 'pguser_from_env'
+        os.environ['PGPASSWORD'] = 'pguser_pwd_from_env'
+        os.environ['PATRONI_RESTAPI_CONNECT_ADDRESS'] = 'localhost:8080'
+        os.environ['PATRONI_RESTAPI_LISTEN'] = 'localhost:8080'
+        os.environ['PATRONI_POSTGRESQL_BIN_POSTGRES'] = 'custom_postgres_bin_from_env'
+
+        self.environ = deepcopy(os.environ)
+
+        dynamic_config = Config.get_default_config()
+        dynamic_config['postgresql']['parameters'] = dict(dynamic_config['postgresql']['parameters'])
+        del dynamic_config['standby_cluster']
+        dynamic_config['postgresql']['parameters']['wal_keep_segments'] = 8
+        dynamic_config['postgresql']['use_pg_rewind'] = True
+
+        self.config = {
+            'scope': self.environ['PATRONI_SCOPE'],
+            'name': self._HOSTNAME,
+            'bootstrap': {
+                'dcs': dynamic_config
+            },
+            'postgresql': {
+                'connect_address': self.no_value_msg + ':5432',
+                'data_dir': self.no_value_msg,
+                'listen': self.no_value_msg + ':5432',
+                'pg_hba': ['host all all all md5',
+                           f'host replication {self.environ["PATRONI_REPLICATION_USERNAME"]} all md5'],
+                'authentication': {'superuser': {'username': self.environ['PATRONI_SUPERUSER_USERNAME'],
+                                                 'password': self.environ['PATRONI_SUPERUSER_PASSWORD']},
+                                   'replication': {'username': self.environ['PATRONI_REPLICATION_USERNAME'],
+                                                   'password': self.environ['PATRONI_REPLICATION_PASSWORD']},
+                                   'rewind': {'username': self.environ['PATRONI_REWIND_USERNAME']}},
+                'bin_dir': self.environ['PATRONI_POSTGRESQL_BIN_DIR'],
+                'bin_name': {'postgres': self.environ['PATRONI_POSTGRESQL_BIN_POSTGRES']},
+                'parameters': {'password_encryption': 'md5'}
+            },
+            'restapi': {
+                'connect_address': self.environ['PATRONI_RESTAPI_CONNECT_ADDRESS'],
+                'listen': self.environ['PATRONI_RESTAPI_LISTEN']
+            }
+        }
+
+    def _set_running_instance_config_vals(self):
+        # values are taken from tests/__init__.py
+        conf = {
+            'scope': 'my_cluster',
+            'bootstrap': {
+                'dcs': {
+                    'postgresql': {
+                        'parameters': {
+                            'max_connections': 42,
+                            'max_locks_per_transaction': 73,
+                            'max_replication_slots': 21,
+                            'max_wal_senders': 37,
+                            'wal_level': 'replica',
+                            'wal_keep_segments': None
+                        },
+                        'use_pg_rewind': None
+                    }
+                }
+            },
+            'postgresql': {
+                'connect_address': f'{self._IP}:bar',
+                'listen': '6.6.6.6:1984',
+                'data_dir': 'data',
+                'bin_dir': '/bin/dir/from/running',
+                'parameters': {
+                    'archive_command': 'my archive command',
+                    'hba_file': os.path.join('data', 'pg_hba.conf'),
+                    'ident_file': os.path.join('data', 'pg_ident.conf'),
+                    'password_encryption': None
+                },
+                'authentication': {
+                    'superuser': {
+                        'username': 'foobar',
+                        'password': 'qwerty',
+                        'channel_binding': 'prefer',
+                        'gssencmode': 'prefer',
+                        'sslmode': 'prefer'
+                    },
+                    'replication': {
+                        'username': self.no_value_msg,
+                        'password': self.no_value_msg
+                    },
+                    'rewind': None
+                },
+            }
+        }
+        patch_config(self.config, conf)
+
+    def _get_running_instance_open_res(self):
+        hba_content = '\n'.join(self.config['postgresql']['pg_hba'] + ['#host all all all md5',
+                                                                       '  host all all all md5',
+                                                                       '',
+                                                                       'hostall all all md5'])
+        ident_content = '\n'.join(['# something very interesting', '  '])
+
+        self.config['postgresql']['pg_hba'] += ['host all all all md5']
+        return [
+            mock_open(read_data=hba_content)(),
+            mock_open(read_data=ident_content)(),
+            mock_open(read_data='1984')(),
+            mock_open()()
+        ]
+
+    @patch('os.makedirs')
+    @patch('yaml.safe_dump')
+    def test_generate_sample_config_pre_13_dir_creation(self, mock_config_dump, mock_makedir):
+        with patch('sys.argv', ['patroni.py', '--generate-sample-config', '/foo/bar.yml']), \
+             patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 9.4.3")) as pg_bin_mock, \
+             self.assertRaises(SystemExit) as e:
+            _main()
+        self.assertEqual(e.exception.code, 0)
+        self.assertEqual(self.config, mock_config_dump.call_args[0][0])
+        mock_makedir.assert_called_once()
+        pg_bin_mock.assert_called_once_with([os.path.join(self.environ['PATRONI_POSTGRESQL_BIN_DIR'],
+                                                          self.environ['PATRONI_POSTGRESQL_BIN_POSTGRES']),
+                                             '--version'])
+
+    @patch('os.makedirs', Mock())
+    @patch('yaml.safe_dump')
+    def test_generate_sample_config_16(self, mock_config_dump):
+        conf = {
+            'bootstrap': {
+                'dcs': {
+                    'postgresql': {
+                        'parameters': {
+                            'wal_keep_size': '128MB',
+                            'wal_keep_segments': None
+                        },
+                    }
+                }
+            },
+            'postgresql': {
+                'parameters': {
+                    'password_encryption': 'scram-sha-256'
+                },
+                'pg_hba': ['host all all all scram-sha-256',
+                           f'host replication {self.environ["PATRONI_REPLICATION_USERNAME"]} all scram-sha-256'],
+                'authentication': {
+                    'rewind': {
+                        'username': self.environ['PATRONI_REWIND_USERNAME'],
+                        'password': self.no_value_msg}
+                },
+            }
+        }
+        patch_config(self.config, conf)
+
+        with patch('sys.argv', ['patroni.py', '--generate-sample-config', '/foo/bar.yml']), \
+             self.assertRaises(SystemExit) as e:
+            _main()
+        self.assertEqual(e.exception.code, 0)
+        self.assertEqual(self.config, mock_config_dump.call_args[0][0])
+
+    @patch('os.makedirs', Mock())
+    @patch('yaml.safe_dump')
+    def test_generate_config_running_instance_16(self, mock_config_dump):
+        self._set_running_instance_config_vals()
+
+        with patch('builtins.open', Mock(side_effect=self._get_running_instance_open_res())), \
+             patch('sys.argv', ['patroni.py', '--generate-config',
+                                '--dsn', 'host=foo port=bar user=foobar password=qwerty']), \
+                self.assertRaises(SystemExit) as e:
+            _main()
+        self.assertEqual(e.exception.code, 0)
+        self.assertEqual(self.config, mock_config_dump.call_args[0][0])
+
+    @patch('os.makedirs', Mock())
+    @patch('yaml.safe_dump')
+    def test_generate_config_running_instance_16_connect_from_env(self, mock_config_dump):
+        self._set_running_instance_config_vals()
+        # su auth params and connect host from env
+        os.environ['PGCHANNELBINDING'] = \
+            self.config['postgresql']['authentication']['superuser']['channel_binding'] = 'disable'
+
+        conf = {
+            'scope': 'my_cluster',
+            'bootstrap': {
+                'dcs': {
+                    'postgresql': {
+                        'parameters': {
+                            'max_connections': 42,
+                            'max_locks_per_transaction': 73,
+                            'max_replication_slots': 21,
+                            'max_wal_senders': 37,
+                            'wal_level': 'replica',
+                            'wal_keep_segments': None
+                        },
+                        'use_pg_rewind': None
+                    }
+                }
+            },
+            'postgresql': {
+                'connect_address': f'{self._IP}:1984',
+                'authentication': {
+                    'superuser': {
+                        'username': self.environ['PGUSER'],
+                        'password': self.environ['PGPASSWORD'],
+                        'gssencmode': None,
+                        'sslmode': None
+                    },
+                },
+            }
+        }
+        patch_config(self.config, conf)
+
+        with patch('builtins.open', Mock(side_effect=self._get_running_instance_open_res())), \
+             patch('sys.argv', ['patroni.py', '--generate-config']), \
+             patch.object(MockConnect, 'server_version', PropertyMock(return_value=160000)), \
+             self.assertRaises(SystemExit) as e:
+            _main()
+        self.assertEqual(e.exception.code, 0)
+        self.assertEqual(self.config, mock_config_dump.call_args[0][0])
+
+    def test_generate_config_running_instance_errors(self):
+        # 1. Wrong DSN format
+        with patch('sys.argv', ['patroni.py', '--generate-config', '--dsn', 'host:foo port:bar user:foobar']), \
+             self.assertRaises(SystemExit) as e:
+            _main()
+        self.assertIn('Failed to parse DSN string', e.exception.code)
+
+        # 2. User is not a superuser
+        with patch('sys.argv', ['patroni.py',
+                                '--generate-config', '--dsn', 'host=foo port=bar user=foobar password=pwd_from_dsn']), \
+             patch.object(MockCursor, 'rowcount', PropertyMock(return_value=0), create=True), \
+             patch.object(MockConnectionInfo, 'parameter_status', Mock(return_value='off')), \
+             self.assertRaises(SystemExit) as e:
+            _main()
+        self.assertIn('The provided user does not have superuser privilege', e.exception.code)
+
+        # 3. Error while calling postgres --version
+        with patch('subprocess.check_output', Mock(side_effect=OSError)), \
+             patch('sys.argv', ['patroni.py', '--generate-sample-config']), \
+             self.assertRaises(SystemExit) as e:
+            _main()
+        self.assertIn('Failed to get postgres version:', e.exception.code)
+
+        with patch('sys.argv', ['patroni.py', '--generate-config']):
+
+            # 4. empty postmaster.pid
+            with patch('builtins.open', Mock(side_effect=[mock_open(read_data='hba_content')(),
+                                                          mock_open(read_data='ident_content')(),
+                                                          mock_open(read_data='')()])), \
+                 self.assertRaises(SystemExit) as e:
+                _main()
+            self.assertIn('Failed to obtain postmaster pid from postmaster.pid file', e.exception.code)
+
+            # 5. Failed to open postmaster.pid
+            with patch('builtins.open', Mock(side_effect=[mock_open(read_data='hba_content')(),
+                                                          mock_open(read_data='ident_content')(),
+                                                          OSError])), \
+                 self.assertRaises(SystemExit) as e:
+                _main()
+            self.assertIn('Error while reading postmaster.pid file', e.exception.code)
+
+            # 6. Invalid postmaster pid
+            with patch('builtins.open', Mock(side_effect=[mock_open(read_data='hba_content')(),
+                                                          mock_open(read_data='ident_content')(),
+                                                          mock_open(read_data='1984')()])), \
+                 patch('psutil.Process.__init__', Mock(return_value=None)), \
+                 patch('psutil.Process.exe', Mock(side_effect=psutil.NoSuchProcess(1984))), \
+                 self.assertRaises(SystemExit) as e:
+                _main()
+            self.assertIn("Obtained postmaster pid doesn't exist", e.exception.code)
+
+            # 7. Failed to open pg_hba
+            with patch('builtins.open', Mock(side_effect=OSError)), \
+                 self.assertRaises(SystemExit) as e:
+                _main()
+            self.assertIn('Failed to read pg_hba.conf', e.exception.code)
+
+            # 8. Failed to open pg_ident
+            with patch('builtins.open', Mock(side_effect=[mock_open(read_data='hba_content')(), OSError])), \
+                 self.assertRaises(SystemExit) as e:
+                _main()
+            self.assertIn('Failed to read pg_ident.conf', e.exception.code)
+
+            # 9. Failed PG connecttion
+            from . import psycopg
+            with patch('patroni.psycopg.connect', side_effect=psycopg.Error), \
+                 self.assertRaises(SystemExit) as e:
+                _main()
+            self.assertIn('Failed to establish PostgreSQL connection', e.exception.code)
+
+            # 10. An unexpected error
+            with patch.object(AbstractConfigGenerator, '__init__', side_effect=psycopg.Error), \
+                 self.assertRaises(SystemExit) as e:
+                _main()
+            self.assertIn('Unexpected exception', e.exception.code)
+
+    def test_get_address(self):
+        with patch('socket.getaddrinfo', Mock(side_effect=Exception)), \
+             patch('logging.warning') as mock_warning:
+            self.assertEqual(get_address(), (self.no_value_msg, self.no_value_msg))
+            self.assertIn('Failed to obtain address: %r', mock_warning.call_args_list[0][0])

tests/test_consul.py

@@ -197,9 +197,10 @@ class TestConsul(unittest.TestCase):
 
     @patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
     def test_delete_leader(self):
-        self.c.delete_leader()
+        leader = self.c.get_cluster().leader
+        self.c.delete_leader(leader)
         self.c._name = 'other'
-        self.c.delete_leader()
+        self.c.delete_leader(leader)
 
     @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
     def test_initialize(self):

tests/test_ctl.py

@@ -6,12 +6,14 @@ import unittest
 from click.testing import CliRunner
 from datetime import datetime, timedelta
 from mock import patch, Mock, PropertyMock
+from patroni.config import GlobalConfig
 from patroni.ctl import ctl, load_config, output_members, get_dcs, parse_dcs, \
     get_all_members, get_any_member, get_cursor, query_member, PatroniCtlException, apply_config_changes, \
     format_config_for_editing, show_diff, invoke_editor, format_pg_version, CONFIG_FILE_PATH, PatronictlPrettyTable
 from patroni.dcs.etcd import AbstractEtcdClientWithFailover, Cluster, Failover
+from patroni.manual_failover import ManualFailoverPrecheckStatus
 from patroni.psycopg import OperationalError
-from patroni.utils import tzutc
+from patroni.utils import ParseScheduleErrors, tzutc
 from prettytable import PrettyTable, ALL
 from urllib3 import PoolManager
 
@@ -21,13 +23,24 @@ from .test_ha import get_cluster_initialized_without_leader, get_cluster_initial
     get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader, get_cluster, Member
 
 
-@patch('patroni.ctl.load_config', Mock(return_value={
+DEFAULT_CONFIG = {
-    'scope': 'alpha', 'restapi': {'listen': '::', 'certfile': 'a'},
+    'scope': 'alpha',
-    'etcd': {'host': 'localhost:2379'}, 'citus': {'database': 'citus', 'group': 0},
+    'restapi': {'listen': '::', 'certfile': 'a'},
-    'postgresql': {'data_dir': '.', 'pgpass': './pgpass', 'parameters': {}, 'retry_timeout': 5}}))
+    'ctl': {'certfile': 'a'},
+    'etcd': {'host': 'localhost:2379'},
+    'citus': {'database': 'citus', 'group': 0},
+    'postgresql': {'data_dir': '.', 'pgpass': './pgpass', 'parameters': {}, 'retry_timeout': 5}
+}
+
+
+@patch('patroni.ctl.load_config', Mock(return_value=DEFAULT_CONFIG))
 class TestCtl(unittest.TestCase):
     TEST_ROLES = ('master', 'primary', 'leader')
 
+    SCHEDULED_TS = '2055-01-01T12:00:00+01:00'
+    SCHEDULED_TS_NO_TZ = '2055-01-01T12:00:00'
+    SCHEDULED_TS_INVALID = '2055-02-30T12:00:00'
+
     @patch('socket.getaddrinfo', socket_getaddrinfo)
     @patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379']))
     def setUp(self):
@@ -83,100 +96,207 @@ class TestCtl(unittest.TestCase):
         scheduled_at = datetime.now(tzutc) + timedelta(seconds=600)
         cluster = get_cluster_initialized_with_leader(Failover(1, 'foo', 'bar', scheduled_at))
         del cluster.members[1].data['conn_url']
-        for fmt in ('pretty', 'json', 'yaml', 'tsv', 'topology'):
+        for fmt in ('pretty', 'json', 'yaml', 'topology'):
             self.assertIsNone(output_members({}, cluster, name='abc', fmt=fmt))
 
+        with patch('click.echo') as mock_echo:
+            self.assertIsNone(output_members({}, cluster, name='abc', fmt='tsv'))
+            self.assertEqual(mock_echo.call_args[0][0], 'abc\tother\t\tReplica\trunning\t\tunknown')
+
     @patch('patroni.ctl.get_dcs')
     @patch.object(PoolManager, 'request', Mock(return_value=MockResponse()))
     def test_switchover(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
         mock_get_dcs.return_value.set_failover_value = Mock()
+
+        # Confirm
         result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
-        assert 'leader' in result.output
+        self.assertEqual(result.exit_code, 0)
 
-        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
+        # Abort
-                                    input='leader\nother\n2300-01-01T12:23:00\ny')
-        assert result.exit_code == 0
-
-        with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
-            result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
-                                              '--force', '--scheduled', '2015-01-01T12:00:00'])
-            assert result.exit_code == 1
-
-        # Aborting switchover, as we answer NO to the confirmation
         result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\nN')
-        assert result.exit_code == 1
+        self.assertEqual(result.exit_code, 1)
-
-        # Aborting scheduled switchover, as we answer NO to the confirmation
-        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
-                                    '--scheduled', '2015-01-01T12:00:00+01:00'], input='leader\nother\n\nN')
-        assert result.exit_code == 1
-
-        # Target and source are equal
-        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nleader\n\ny')
-        assert result.exit_code == 1
-
-        # Reality is not part of this cluster
-        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nReality\n\ny')
-        assert result.exit_code == 1
 
+        # Without a candidate with --force option
         result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force'])
-        assert 'Member' in result.output
+        self.assertEqual(result.exit_code, 0)
-
-        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
-                                          '--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
-        assert result.exit_code == 0
-
-        # Invalid timestamp
-        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force', '--scheduled', 'invalid'])
-        assert result.exit_code != 0
-
-        # Invalid timestamp
-        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
-                                          '--force', '--scheduled', '2115-02-30T12:00:00+01:00'])
-        assert result.exit_code != 0
-
-        # Specifying wrong leader
-        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='dummy')
-        assert result.exit_code == 1
-
-        with patch.object(PoolManager, 'request', Mock(side_effect=Exception)):
-            # Non-responding patroni
-            result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
-                                        input='leader\nother\n2300-01-01T12:23:00\ny')
-            assert 'falling back to DCS' in result.output
-
-        with patch.object(PoolManager, 'request') as mocked:
-            mocked.return_value.status = 500
-            result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
-            assert 'Switchover failed' in result.output
-
-            mocked.return_value.status = 501
-            mocked.return_value.data = b'Server does not support this operation'
-            result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
-            assert 'Switchover failed' in result.output
 
         # No members available
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
         result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
-        assert result.exit_code == 1
+        self.assertEqual(result.exit_code, 1)
+        self.assertIn('No candidates found to switchover to', result.output)
 
         # No leader available
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
         result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
-        assert result.exit_code == 1
+        self.assertEqual(result.exit_code, 1)
+        self.assertIn('This cluster has no leader', result.output)
+
+        # Citus cluster, no group number specified
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force'], input='\n')
+        self.assertEqual(result.exit_code, 1)
+        self.assertIn('For Citus clusters the --group must me specified', result.output)
+
+        # [Scheduled]
+
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+
+        # Scheduled (confirm)
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
+                                    input=f'leader\nother\n{self.SCHEDULED_TS}\ny')
+        self.assertEqual(result.exit_code, 0)
+        self.assertIn(f'Are you sure you want to schedule a switchover in the cluster dummy '
+                      f'at {self.SCHEDULED_TS}, demoting current leader', result.output)
+
+        # Scheduled (abort)
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
+                                    '--scheduled', self.SCHEDULED_TS], input='leader\nother\n\nN')
+        self.assertEqual(result.exit_code, 1)
+
+        # Scheduled with --force option
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
+                                    '--force', '--scheduled', self.SCHEDULED_TS])
+        self.assertEqual(result.exit_code, 0)
+
+        # Scheduled in pause mode
+        with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
+            result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
+                                              '--force', '--scheduled', self.SCHEDULED_TS])
+            self.assertEqual(result.exit_code, 1)
+            self.assertIn(ManualFailoverPrecheckStatus.SCHEDULED_SWITCHOVER_PAUSE.value[0], result.output)
+
+        # Invalid timestamp with force
+        result = self.runner.invoke(ctl,['switchover', 'dummy', '--group', '0', '--force', '--scheduled',
+                                         self.SCHEDULED_TS_INVALID])
+        self.assertEqual(result.exit_code, 1)
+        self.assertIn('Unable to parse scheduled timestamp', result.output)
+
+        # Invalid timestamp
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
+                                          '--force', '--scheduled', self.SCHEDULED_TS_INVALID])
+        self.assertEqual(result.exit_code, 1)
+        self.assertIn('Unable to parse scheduled timestamp', result.output)
+
+        # Invalid timestamp - no timezone
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
+                                          '--force', '--scheduled', self.SCHEDULED_TS_NO_TZ])
+        self.assertEqual(result.exit_code, 1)
+        self.assertIn(ParseScheduleErrors.NO_TIMEZONE.value[0].format(action='switchover'), result.output)
+
+        # [Other erroneous combinations]
+
+        # No candidate in pause mode
+        with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
+            result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\n\n\ny')
+            self.assertEqual(result.exit_code, 1)
+            self.assertIn(ManualFailoverPrecheckStatus.SWITCHOVER_PAUSE_NO_CANDIDATE.value[0], result.output)
+
+        # Target and source are equal
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nleader\n\ny')
+        self.assertEqual(result.exit_code, 1)
+        self.assertIn(ManualFailoverPrecheckStatus.SWITCHOVER_TO_LEADER.value[0], result.output)
+
+        # Candidate is not a member of the cluster
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nReality\n\ny')
+        self.assertEqual(result.exit_code, 1)
+        self.assertIn(ManualFailoverPrecheckStatus.CANDIDATE_NOT_MEMEBER.value[0].format(candidate='Reality',
+                                                                                         cluster_name='dummy'),
+                      result.output)
+
+        # Specifying wrong leader
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='dummy')
+        self.assertEqual(result.exit_code, 1)
+        self.assertIn(
+            ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER.value[0].format(leader='dummy',
+                                                                           cluster_name='dummy'),
+            result.output)
+
+        mock_get_dcs.return_value.get_cluster = Mock(
+            return_value=get_cluster_initialized_with_leader(sync=('leader', 'other')))
+
+        # Candidate is not a sync standby
+        with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=True)):
+            result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\notherMember\n\ny')
+            self.assertEqual(result.exit_code, 1)
+            self.assertIn(ManualFailoverPrecheckStatus.CANDIDATE_NOT_SYNC_STANDBY.value[0], result.output)
+
+        # No healthy nodes to promote in sync mode
+        mock_get_dcs.return_value.get_cluster = Mock(return_value=get_cluster_initialized_with_leader(sync=('leader')))
+        with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=True)):
+            result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force'])
+            self.assertEqual(result.exit_code, 1)
+            self.assertIn(ManualFailoverPrecheckStatus.NO_SYNC_CANDIDATE.value[0].format(action='switchover'),
+                          result.output)
+
+        # No healthy nodes to promote in async mode
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
+        with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=False)):
+            result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force'])
+            self.assertEqual(result.exit_code, 1)
+            self.assertIn(ManualFailoverPrecheckStatus.ONLY_LEADER.value[0].format(action='switchover'),
+                          result.output)
+
+        # Cluster has no leader
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--leader', 'leader', '--force'])
+        self.assertEqual(result.exit_code, 1)
+        self.assertIn(
+            ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER.value[0].format(leader='leader', cluster_name='dummy'),
+            result.output)
+
+        # [Errors while sending Patroni REST API request]
+
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+
+        with patch.object(PoolManager, 'request', Mock(side_effect=Exception)):
+            result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
+                                        input=f'leader\nother\n{self.SCHEDULED_TS}\ny')
+            self.assertIn('falling back to DCS', result.output)
+
+        with patch.object(PoolManager, 'request') as mock_api_request:
+            mock_api_request.return_value.status = 500
+            result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
+            self.assertIn('Switchover failed', result.output)
+
+            mock_api_request.return_value.status = 501
+            mock_api_request.return_value.data = b'Server does not support this operation'
+            result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
+            self.assertIn('Switchover failed', result.output)
 
     @patch('patroni.ctl.get_dcs')
     @patch.object(PoolManager, 'request', Mock(return_value=MockResponse()))
+    @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
     def test_failover(self, mock_get_dcs):
-        mock_get_dcs.return_value = self.e
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
         mock_get_dcs.return_value.set_failover_value = Mock()
-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force'], input='\n')
+
-        assert 'For Citus clusters the --group must me specified' in result.output
+        # No candidate specified
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
         result = self.runner.invoke(ctl, ['failover', 'dummy'], input='0\n')
-        assert 'Failover could be performed only to a specific candidate' in result.output
+        self.assertIn(ManualFailoverPrecheckStatus.FAILOVER_NO_CANDIDATE.value[0], result.output)
+
+        # Failover to an async member in sync mode (confirm)
+        cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
+
+        # Temp test to check a fallback to switchover if leader is specified
+        with patch('patroni.ctl._do_failover_or_switchover') as failover_func_mock:
+            result = self.runner.invoke(ctl, ['failover', '--leader', 'leader', 'dummy'], input='0\n')
+            self.assertIn('Supplying a leader name using this command is deprecated', result.output)
+            failover_func_mock.assert_called_once_with(
+                DEFAULT_CONFIG, 'switchover', 'dummy', None, 'leader', None, False)
+
+        # Failover to an async member in sync mode (confirm)
+        cluster.members.append(Member(0, 'async', 28, {'api_url': 'http://127.0.0.1:8012/patroni'}))
+        cluster.config.data['synchronous_mode'] = True
+        mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
+        result = self.runner.invoke(ctl, ['failover', 'dummy', '--group', '0', '--candidate', 'async'], input='y\ny')
+        self.assertIn('Are you sure you want to failover to the asynchronous node async', result.output)
+
+        # Failover to an async member in sync mode (abort)
+        mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
+        result = self.runner.invoke(ctl, ['failover', 'dummy', '--group', '0', '--candidate', 'async'], input='N')
+        self.assertEqual(result.exit_code, 1)
 
     @patch('patroni.dcs.dcs_modules', Mock(return_value=['patroni.dcs.dummy', 'patroni.dcs.etcd']))
     def test_get_dcs(self):
@@ -269,12 +389,9 @@ class TestCtl(unittest.TestCase):
 
     @patch.object(PoolManager, 'request')
     @patch('patroni.ctl.get_dcs')
-    def test_restart_reinit(self, mock_get_dcs, mock_post):
+    def test_reinit(self, mock_get_dcs, mock_post):
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
         mock_post.return_value.status = 503
-        result = self.runner.invoke(ctl, ['restart', 'alpha'], input='now\ny\n')
-        assert 'Failed: restart for' in result.output
-        assert result.exit_code == 0
 
         result = self.runner.invoke(ctl, ['reinit', 'alpha'], input='y')
         assert result.exit_code == 1
@@ -283,67 +400,88 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['reinit', 'alpha', 'other'], input='y\ny')
         assert result.exit_code == 0
 
-        # Aborted restart
+    @patch.object(PoolManager, 'request')
+    @patch('patroni.ctl.get_dcs')
+    def test_restart(self, mock_get_dcs, mock_post):
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        mock_post.return_value.status = 200
+
+        # Successful restart
+        result = self.runner.invoke(ctl, ['restart', 'alpha'], input='now\ny\n')
+        self.assertEqual(result.exit_code, 0)
+
+        # Aborted
         result = self.runner.invoke(ctl, ['restart', 'alpha'], input='now\nN')
-        assert result.exit_code == 1
+        self.assertEqual(result.exit_code, 1)
 
+        # With pending the flag
         result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force'])
-        assert result.exit_code == 0
+        self.assertEqual(result.exit_code, 0)
-
-        # Aborted scheduled restart
-        result = self.runner.invoke(ctl, ['restart', 'alpha', '--scheduled', '2019-10-01T14:30'], input='N')
-        assert result.exit_code == 1
 
         # Not a member
         result = self.runner.invoke(ctl, ['restart', 'alpha', 'dummy', '--any'], input='now\ny')
-        assert result.exit_code == 1
+        self.assertEqual(result.exit_code, 1)
+        self.assertIn('Not a single cluster member among provided members', result.output)
+
+        # Not a member with the specified role
+        result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--role', 'primary'], input='now\ny')
+        self.assertEqual(result.exit_code, 1)
+        self.assertIn('No primary among provided members', result.output)
 
         # Wrong pg version
         result = self.runner.invoke(ctl, ['restart', 'alpha', '--any', '--pg-version', '9.1'], input='now\ny')
-        assert 'Error: Invalid PostgreSQL version format' in result.output
+        self.assertEqual(result.exit_code, 1)
-        assert result.exit_code == 1
+        self.assertIn('Error: Invalid PostgreSQL version format', result.output)
 
+        # Restart with timeout
         result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force', '--timeout', '10min'])
-        assert result.exit_code == 0
+        self.assertEqual(result.exit_code, 0)
 
-        # normal restart, the schedule is actually parsed, but not validated in patronictl
+        # Scheduled restart
-        result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
-        assert 'Failed: flush scheduled restart' in result.output
 
+        # Aborted scheduled restart
+        result = self.runner.invoke(ctl, ['restart', 'alpha', '--scheduled', self.SCHEDULED_TS], input='N')
+        self.assertEqual(result.exit_code, 1)
+
+        # Error parsing scheduled flag value (no tz)
+        result = self.runner.invoke(ctl,
+                                    ['restart', 'alpha', 'other', '--force', '--scheduled', self.SCHEDULED_TS_NO_TZ])
+        self.assertEqual(result.exit_code, 1)
+        self.assertIn(ParseScheduleErrors.NO_TIMEZONE.value[0].format(action='restart'), result.output)
+
+        # Error parsing scheduled flag value (invalid date)
+        result = self.runner.invoke(ctl,
+                                    ['restart', 'alpha', 'other', '--force', '--scheduled', self.SCHEDULED_TS_INVALID])
+        self.assertEqual(result.exit_code, 1)
+        self.assertIn('Unable to parse scheduled timestamp', result.output)
+
+        # Successfully scheduled restart
+        result = self.runner.invoke(ctl, ['restart', 'alpha', '--scheduled', self.SCHEDULED_TS], input='Y')
+        self.assertEqual(result.exit_code, 0)
+        self.assertIn('Success: restart on member other', result.output)
+
+        # Not possible to schedule in pause mode
         with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
             result = self.runner.invoke(ctl,
-                                        ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
+                                        ['restart', 'alpha', 'other', '--force', '--scheduled', self.SCHEDULED_TS])
-            assert result.exit_code == 1
+            self.assertEqual(result.exit_code, 1)
+            self.assertIn("Can't schedule restart in the paused state", result.output)
 
-        # force restart with restart already present
+        # Force restart with restart already scheduled
-        result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
+        result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force', '--scheduled', self.SCHEDULED_TS])
-        assert result.exit_code == 0
+        self.assertEqual(result.exit_code, 0)
-
-        ctl_args = ['restart', 'alpha', '--pg-version', '99.0', '--scheduled', '2300-10-01T14:30']
-        # normal restart, the schedule is actually parsed, but not validated in patronictl
-        mock_post.return_value.status = 200
-        result = self.runner.invoke(ctl, ctl_args, input='y')
-        assert result.exit_code == 0
 
         # get restart with the non-200 return code
-        # normal restart, the schedule is actually parsed, but not validated in patronictl
+        ctl_args = ['restart', 'alpha', '--pg-version', '99.0', '--scheduled', self.SCHEDULED_TS]
-        mock_post.return_value.status = 204
+        for code, output in [
-        result = self.runner.invoke(ctl, ctl_args, input='y')
+            (204, 'Failed: restart for member other, status code=204'),
-        assert result.exit_code == 0
+            (202, 'Success: restart scheduled'),
-
+            (409, 'Failed: another restart is already')
-        # get restart with the non-200 return code
+        ]:
-        # normal restart, the schedule is actually parsed, but not validated in patronictl
+            mock_post.return_value.status = code
-        mock_post.return_value.status = 202
+            result = self.runner.invoke(ctl, ctl_args, input='y')
-        result = self.runner.invoke(ctl, ctl_args, input='y')
+            self.assertEqual(result.exit_code, 0)
-        assert 'Success: restart scheduled' in result.output
+            self.assertIn(output, result.output)
-        assert result.exit_code == 0
-
-        # get restart with the non-200 return code
-        # normal restart, the schedule is actually parsed, but not validated in patronictl
-        mock_post.return_value.status = 409
-        result = self.runner.invoke(ctl, ctl_args, input='y')
-        assert 'Failed: another restart is already' in result.output
-        assert result.exit_code == 0
 
     @patch('patroni.ctl.get_dcs')
     def test_remove(self, mock_get_dcs):
@@ -398,9 +536,19 @@ class TestCtl(unittest.TestCase):
     @patch('patroni.ctl.get_dcs')
     def test_members(self, mock_get_dcs):
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+
         result = self.runner.invoke(ctl, ['list'])
         assert '127.0.0.1' in result.output
         assert result.exit_code == 0
+        assert 'Citus cluster: alpha -' in result.output
+
+        result = self.runner.invoke(ctl, ['list', '--group', '0'])
+        assert 'Citus cluster: alpha (group: 0, 12345678901) -' in result.output
+
+        with patch('patroni.ctl.load_config', Mock(return_value={'scope': 'alpha'})):
+            result = self.runner.invoke(ctl, ['list'])
+            assert 'Cluster: alpha (12345678901) -' in result.output
+
         with patch('patroni.ctl.load_config', Mock(return_value={})):
             self.runner.invoke(ctl, ['list'])
 
@@ -451,7 +599,7 @@ class TestCtl(unittest.TestCase):
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
 
         for role in self.TEST_ROLES:
-            result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '-r', role], input='y')
+            result = self.runner.invoke(ctl, ['-k', 'flush', 'dummy', 'restart', '-r', role], input='y')
             assert 'No scheduled restart' in result.output
 
         result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force'])

tests/test_etcd.py

@@ -172,12 +172,12 @@ class TestClient(unittest.TestCase):
         self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
         self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
 
-        with patch.object(EtcdClient, '_calculate_timeouts', Mock(side_effect=[(1, 1, 0), (1, 1, 0), (0, 1, 0)])),\
+        with patch.object(EtcdClient, '_calculate_timeouts', Mock(side_effect=[(1, 1, 0), (1, 1, 0), (0, 1, 0)])), \
                 patch.object(EtcdClient, '_load_machines_cache', Mock(side_effect=Exception)):
             self.client.http.request = Mock(side_effect=socket.error)
             self.assertRaises(etcd.EtcdException, rtry, self.client.api_execute, '/', 'GET', params={'retry': rtry})
 
-        with patch.object(EtcdClient, '_calculate_timeouts', Mock(side_effect=[(1, 1, 0), (1, 1, 0), (0, 1, 0)])),\
+        with patch.object(EtcdClient, '_calculate_timeouts', Mock(side_effect=[(1, 1, 0), (1, 1, 0), (0, 1, 0)])), \
                 patch.object(EtcdClient, '_load_machines_cache', Mock(return_value=True)):
             self.assertRaises(etcd.EtcdException, rtry, self.client.api_execute, '/', 'GET', params={'retry': rtry})
 
@@ -313,7 +313,7 @@ class TestEtcd(unittest.TestCase):
         self.assertFalse(self.etcd.cancel_initialization())
 
     def test_delete_leader(self):
-        self.assertFalse(self.etcd.delete_leader())
+        self.assertFalse(self.etcd.delete_leader(self.etcd.get_cluster().leader))
 
     def test_delete_cluster(self):
         self.assertFalse(self.etcd.delete_cluster())

tests/test_etcd3.py

@@ -5,8 +5,9 @@ import urllib3
 
 from mock import Mock, PropertyMock, patch
 from patroni.dcs.etcd import DnsCachingResolver
-from patroni.dcs.etcd3 import PatroniEtcd3Client, Cluster, Etcd3Client, Etcd3Error, Etcd3ClientError, RetryFailedError,\
+from patroni.dcs.etcd3 import PatroniEtcd3Client, Cluster, Etcd3, Etcd3Client, \
-    InvalidAuthToken, Unavailable, Unknown, UnsupportedEtcdVersion, UserEmpty, AuthFailed, base64_encode, Etcd3
+    Etcd3Error, Etcd3ClientError, RetryFailedError, InvalidAuthToken, Unavailable, \
+    Unknown, UnsupportedEtcdVersion, UserEmpty, AuthFailed, base64_encode
 from threading import Thread
 
 from . import SleepException, MockResponse
@@ -126,10 +127,16 @@ class TestPatroniEtcd3Client(BaseTestEtcd3):
         request = {'key': base64_encode('/patroni/test/leader')}
         mock_urlopen.return_value = MockResponse()
         mock_urlopen.return_value.content = '{"succeeded":true,"header":{"revision":"1"}}'
-        self.client.call_rpc('/kv/txn', {'success': [{'request_delete_range': request}]})
         self.client.call_rpc('/kv/put', request)
         self.client.call_rpc('/kv/deleterange', request)
 
+    @patch.object(urllib3.PoolManager, 'urlopen')
+    def test_txn(self, mock_urlopen):
+        mock_urlopen.return_value = MockResponse()
+        mock_urlopen.return_value.content = '{"header":{"revision":"1"}}'
+        self.client.txn({'target': 'MOD', 'mod_revision': '1'},
+                        {'request_delete_range': {'key': base64_encode('/patroni/test/leader')}})
+
     @patch('time.time', Mock(side_effect=[1, 10.9, 100]))
     def test__wait_cache(self):
         with self.kv_cache.condition:
@@ -241,7 +248,7 @@ class TestEtcd3(BaseTestEtcd3):
             self.etcd3.update_leader(leader, '123', failsafe={'foo': 'bar'})
         self.etcd3._last_lease_refresh = 0
         self.etcd3.update_leader(leader, '124')
-        with patch.object(PatroniEtcd3Client, 'lease_keepalive', Mock(return_value=True)),\
+        with patch.object(PatroniEtcd3Client, 'lease_keepalive', Mock(return_value=True)), \
                 patch('time.time', Mock(side_effect=[0, 100, 200, 300])):
             self.assertRaises(Etcd3Error, self.etcd3.update_leader, leader, '126')
         self.etcd3._lease = leader.session
@@ -291,9 +298,10 @@ class TestEtcd3(BaseTestEtcd3):
         self.etcd3.cancel_initialization()
 
     def test_delete_leader(self):
-        self.etcd3.delete_leader()
+        leader = self.etcd3.get_cluster().leader
+        self.etcd3.delete_leader(leader)
         self.etcd3._name = 'other'
-        self.etcd3.delete_leader()
+        self.etcd3.delete_leader(leader)
 
     def test_delete_cluster(self):
         self.etcd3.delete_cluster()
@@ -305,7 +313,7 @@ class TestEtcd3(BaseTestEtcd3):
         self.etcd3.set_sync_state_value('', 1)
 
     def test_delete_sync_state(self):
-        self.etcd3.delete_sync_state()
+        self.etcd3.delete_sync_state('1')
 
     def test_watch(self):
         self.etcd3.set_ttl(10)

tests/test_file_perm.py

@@ -0,0 +1,33 @@
+import unittest
+import stat
+
+from mock import Mock, patch
+
+from patroni.file_perm import pg_perm
+
+
+class TestFilePermissions(unittest.TestCase):
+
+    @patch('os.stat')
+    @patch('os.umask')
+    @patch('patroni.file_perm.logger.error')
+    def test_set_umask(self, mock_logger, mock_umask, mock_stat):
+        mock_umask.side_effect = Exception
+        mock_stat.return_value.st_mode = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP
+        pg_perm.set_permissions_from_data_directory('test')
+
+        # umask is called with PG_MODE_MASK_GROUP
+        self.assertEqual(mock_umask.call_args[0][0], stat.S_IWGRP | stat.S_IRWXO)
+        self.assertEqual(mock_logger.call_args[0][0], 'Can not set umask to %03o: %r')
+
+        mock_umask.reset_mock()
+        mock_stat.return_value.st_mode = stat.S_IRWXU
+        pg_perm.set_permissions_from_data_directory('test')
+        # umask is called with PG_MODE_MASK_OWNER (permissions changed from group to owner)
+        self.assertEqual(mock_umask.call_args[0][0], stat.S_IRWXG | stat.S_IRWXO)
+
+    @patch('os.stat', Mock(side_effect=FileNotFoundError))
+    @patch('patroni.file_perm.logger.error')
+    def test_set_permissions_from_data_directory(self, mock_logger):
+        pg_perm.set_permissions_from_data_directory('test')
+        self.assertEqual(mock_logger.call_args[0][0], 'Can not check permissions on %s: %r')

tests/test_ha.py

@@ -99,7 +99,9 @@ def get_node_status(reachable=True, in_recovery=True, dcs_last_seen=0,
         tags = {}
         if nofailover:
             tags['nofailover'] = True
-        return _MemberStatus(e, reachable, in_recovery, dcs_last_seen, timeline, wal_position, tags, watchdog_failed)
+        return _MemberStatus(e, reachable, in_recovery, wal_position,
+                             {'tags': tags, 'watchdog_failed': watchdog_failed,
+                              'dcs_last_seen': dcs_last_seen, 'timeline': timeline})
     return fetch_node_status
 
 
@@ -162,7 +164,7 @@ def run_async(self, func, args=()):
 
 
 @patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
-@patch.object(Postgresql, 'is_leader', Mock(return_value=True))
+@patch.object(Postgresql, 'is_primary', Mock(return_value=True))
 @patch.object(Postgresql, 'timeline_wal_position', Mock(return_value=(1, 10, 1)))
 @patch.object(Postgresql, '_cluster_info_state_get', Mock(return_value=10))
 @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
@@ -224,7 +226,7 @@ class TestHa(PostgresInit):
     @patch.object(Postgresql, 'received_timeline', Mock(return_value=None))
     def test_touch_member(self):
         self.p._major_version = 110000
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.p.timeline_wal_position = Mock(return_value=(0, 1, 0))
         self.p.replica_cached_timeline = Mock(side_effect=Exception)
         with patch.object(Postgresql, '_cluster_info_state_get', Mock(return_value='streaming')):
@@ -278,8 +280,10 @@ class TestHa(PostgresInit):
         self.p.follow = true
         self.assertEqual(self.ha.run_cycle(), 'starting as a secondary')
         self.p.is_running = true
+        ha_dcs_orig_name = self.ha.dcs.__class__.__name__
         self.ha.dcs.__class__.__name__ = 'Raft'
         self.assertEqual(self.ha.run_cycle(), 'started as a secondary')
+        self.ha.dcs.__class__.__name__ = ha_dcs_orig_name
 
     def test_recover_former_primary(self):
         self.p.follow = false
@@ -305,7 +309,7 @@ class TestHa(PostgresInit):
         self.p.is_running = false
         self.p.controldata = lambda: {'Database cluster state': 'in production', 'Database system identifier': SYSID}
         self.assertEqual(self.ha.run_cycle(), 'doing crash recovery in a single user mode')
-        with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)),\
+        with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)), \
                 patch.object(Ha, 'check_timeline', Mock(return_value=False)):
             self.ha._async_executor.schedule('doing crash recovery in a single user mode')
             self.ha.state_handler.cancellable._process = Mock()
@@ -318,7 +322,7 @@ class TestHa(PostgresInit):
     @patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
     @patch.object(Rewind, 'can_rewind', PropertyMock(return_value=True))
     def test_crash_recovery_before_rewind(self):
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.p.is_running = false
         self.p.controldata = lambda: {'Database cluster state': 'in archive recovery',
                                       'Database system identifier': SYSID}
@@ -338,7 +342,7 @@ class TestHa(PostgresInit):
         self.ha._rewind.check_leader_is_not_in_recovery = true
         with patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True)):
             self.assertEqual(self.ha.run_cycle(), 'running pg_rewind from leader')
-        with patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=False)),\
+        with patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=False)), \
                 patch.object(Ha, 'is_synchronous_mode', Mock(return_value=True)):
             self.p.follow = true
             self.assertEqual(self.ha.run_cycle(), 'starting as a secondary')
@@ -363,7 +367,7 @@ class TestHa(PostgresInit):
 
     @patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
     def test_start_as_readonly(self):
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.p.is_healthy = true
         self.ha.has_lock = true
         self.p.controldata = lambda: {'Database cluster state': 'in production', 'Database system identifier': SYSID}
@@ -373,13 +377,19 @@ class TestHa(PostgresInit):
     def test_acquire_lock_as_primary(self):
         self.assertEqual(self.ha.run_cycle(), 'acquired session lock as a leader')
 
+    def test_leader_race_stale_primary(self):
+        with patch.object(Postgresql, 'get_primary_timeline', Mock(return_value=1)), \
+                patch('patroni.ha.logger.warning') as mock_logger:
+            self.assertEqual(self.ha.run_cycle(), 'demoting self because i am not the healthiest node')
+            self.assertEqual(mock_logger.call_args[0][0], 'My timeline %s is behind last known cluster timeline %s')
+
     def test_promoted_by_acquiring_lock(self):
         self.ha.is_healthiest_node = true
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
 
     def test_promotion_cancelled_after_pre_promote_failed(self):
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.p._pre_promote = false
         self.ha._is_healthiest_node = true
         self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
@@ -394,7 +404,7 @@ class TestHa(PostgresInit):
     @patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
     def test_long_promote(self):
         self.ha.has_lock = true
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.p.set_role('primary')
         self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
 
@@ -405,7 +415,7 @@ class TestHa(PostgresInit):
     def test_follow_new_leader_after_failing_to_obtain_lock(self):
         self.ha.is_healthiest_node = true
         self.ha.acquire_lock = false
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.assertEqual(self.ha.run_cycle(), 'following new leader after trying and failing to obtain lock')
 
     def test_demote_because_not_healthiest(self):
@@ -414,21 +424,21 @@ class TestHa(PostgresInit):
 
     def test_follow_new_leader_because_not_healthiest(self):
         self.ha.is_healthiest_node = false
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
 
     @patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
     def test_promote_because_have_lock(self):
         self.ha.has_lock = true
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.assertEqual(self.ha.run_cycle(), 'promoted self to leader because I had the session lock')
 
     def test_promote_without_watchdog(self):
         self.ha.has_lock = true
-        self.p.is_leader = true
+        self.p.is_primary = true
         with patch.object(Watchdog, 'activate', Mock(return_value=False)):
             self.assertEqual(self.ha.run_cycle(), 'Demoting self because watchdog could not be activated')
-            self.p.is_leader = false
+            self.p.is_primary = false
             self.assertEqual(self.ha.run_cycle(), 'Not promoting self because watchdog could not be activated')
 
     def test_leader_with_lock(self):
@@ -454,12 +464,12 @@ class TestHa(PostgresInit):
         self.assertEqual(self.ha.run_cycle(), 'demoted self because failed to update leader lock in DCS')
         with patch.object(Ha, '_get_node_to_follow', Mock(side_effect=DCSError('foo'))):
             self.assertEqual(self.ha.run_cycle(), 'demoted self because failed to update leader lock in DCS')
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.assertEqual(self.ha.run_cycle(), 'not promoting because failed to update leader lock in DCS')
 
     @patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
     def test_follow(self):
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), a secondary, and following a leader ()')
         self.ha.patroni.replicatefrom = "foo"
         self.p.config.check_recovery_conf = Mock(return_value=(True, False))
@@ -476,13 +486,13 @@ class TestHa(PostgresInit):
     def test_follow_in_pause(self):
         self.ha.is_paused = true
         self.assertEqual(self.ha.run_cycle(), 'PAUSE: continue to run as primary without lock')
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.assertEqual(self.ha.run_cycle(), 'PAUSE: no action. I am (postgresql0)')
 
     @patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
     @patch.object(Rewind, 'can_rewind', PropertyMock(return_value=True))
     def test_follow_triggers_rewind(self):
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.ha._rewind.trigger_check_diverged_lsn()
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.assertEqual(self.ha.run_cycle(), 'running pg_rewind from leader')
@@ -536,7 +546,7 @@ class TestHa(PostgresInit):
         self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
         self.ha.update_failsafe({'name': 'leader', 'api_url': 'http://127.0.0.1:8008/patroni',
                                  'conn_url': 'postgres://127.0.0.1:5432/postgres', 'slots': {'foo': 1000}})
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.assertEqual(self.ha.run_cycle(), 'DCS is not accessible')
 
     def test_no_dcs_connection_replica_failsafe_not_enabled_but_active(self):
@@ -544,7 +554,7 @@ class TestHa(PostgresInit):
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.ha.update_failsafe({'name': 'leader', 'api_url': 'http://127.0.0.1:8008/patroni',
                                  'conn_url': 'postgres://127.0.0.1:5432/postgres', 'slots': {'foo': 1000}})
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.assertEqual(self.ha.run_cycle(), 'DCS is not accessible')
 
     def test_update_failsafe(self):
@@ -583,9 +593,9 @@ class TestHa(PostgresInit):
         self.ha.cluster = get_cluster_not_initialized_without_leader()
         self.e.initialize = true
         self.assertEqual(self.ha.bootstrap(), 'trying to bootstrap a new cluster')
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.assertEqual(self.ha.run_cycle(), 'waiting for end of recovery after bootstrap')
-        self.p.is_leader = true
+        self.p.is_primary = true
         self.ha.is_synchronous_mode = true
         self.assertEqual(self.ha.run_cycle(), 'running post_bootstrap')
         self.assertEqual(self.ha.run_cycle(), 'initialized a new cluster')
@@ -605,8 +615,8 @@ class TestHa(PostgresInit):
         self.ha.cluster = get_cluster_not_initialized_without_leader()
         self.e.initialize = true
         self.ha.bootstrap()
-        self.p.is_leader = true
+        self.p.is_primary = true
-        with patch.object(Watchdog, 'activate', Mock(return_value=False)),\
+        with patch.object(Watchdog, 'activate', Mock(return_value=False)), \
                 patch('patroni.ha.logger.error') as mock_logger:
             self.assertEqual(self.ha.post_bootstrap(), 'running post_bootstrap')
             self.assertRaises(PatroniFatalException, self.ha.post_bootstrap)
@@ -667,9 +677,9 @@ class TestHa(PostgresInit):
 
             self.ha.update_lock = false
             self.p.set_role('primary')
-            with patch('patroni.async_executor.CriticalTask.cancel', Mock(return_value=False)),\
+            with patch('patroni.async_executor.CriticalTask.cancel', Mock(return_value=False)), \
                     patch('patroni.async_executor.CriticalTask.result',
-                          PropertyMock(return_value=PostmasterProcess(os.getpid())), create=True),\
+                          PropertyMock(return_value=PostmasterProcess(os.getpid())), create=True), \
                     patch('patroni.postgresql.Postgresql.terminate_starting_postmaster') as mock_terminate:
                 self.assertEqual(self.ha.run_cycle(), 'lost leader lock during restart')
                 mock_terminate.assert_called()
@@ -679,112 +689,289 @@ class TestHa(PostgresInit):
 
     @patch('patroni.postgresql.citus.CitusHandler.is_coordinator', Mock(return_value=False))
     def test_manual_failover_from_leader(self):
+        self.ha.has_lock = true  # I am the leader
+
+        # to me
+        with patch('patroni.ha.logger.warning') as mock_warning:
+            self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None))
+            self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
+            mock_warning.assert_called_with('%s: I am already the leader, no need to %s', 'manual failover', 'failover')
+
+        # to a non-existent candidate
+        with patch('patroni.ha.logger.warning') as mock_warning:
+            self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla', None))
+            self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
+            mock_warning.assert_called_with(
+                '%s: no healthy members found, %s is not possible', 'manual failover', 'failover')
+
+        # to an existent candidate
         self.ha.fetch_node_status = get_node_status()
-        self.ha.has_lock = true
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'b', None))
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None))
+        self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
-        self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None))
-        self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla', None))
-        self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
-        f = Failover(0, self.p.name, '', None)
-        self.ha.cluster = get_cluster_initialized_with_leader(f)
         self.assertEqual(self.ha.run_cycle(), 'manual failover: demoting myself')
+
+        # to a candidate on an older timeline
+        with patch('patroni.ha.logger.info') as mock_info:
+            self.ha.fetch_node_status = get_node_status(timeline=1)
+            self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
+            self.assertEqual(mock_info.call_args_list[0][0],
+                             ('Timeline %s of member %s is behind the cluster timeline %s', 1, 'b', 2))
+
+        # to a lagging candidate
+        with patch('patroni.ha.logger.info') as mock_info:
+            self.ha.fetch_node_status = get_node_status(wal_position=1)
+            self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5})
+            self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
+            self.assertEqual(mock_info.call_args_list[0][0],
+                             ('Member %s exceeds maximum replication lag', 'b'))
+            self.ha.cluster.members.pop()
+
+    @patch('patroni.postgresql.citus.CitusHandler.is_coordinator', Mock(return_value=False))
+    def test_manual_switchover_from_leader(self):
+        self.ha.has_lock = true  # I am the leader
+
+        self.ha.fetch_node_status = get_node_status()
+
+        # different leader specified in failover key, no candidate
+        with patch('patroni.ha.logger.warning') as mock_warning:
+            self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None))
+            self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
+            mock_warning.assert_called_with(
+                '%s: leader name does not match: %s != %s', 'switchover', 'blabla', 'postgresql0')
+
+        # no candidate
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, '', None))
+        self.assertEqual(self.ha.run_cycle(), 'switchover: demoting myself')
+
         self.ha._rewind.rewind_or_reinitialize_needed_and_possible = true
-        self.assertEqual(self.ha.run_cycle(), 'manual failover: demoting myself')
+        self.assertEqual(self.ha.run_cycle(), 'switchover: demoting myself')
-        self.ha.fetch_node_status = get_node_status(nofailover=True)
-        self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
-        self.ha.fetch_node_status = get_node_status(watchdog_failed=True)
-        self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
-        self.ha.fetch_node_status = get_node_status(timeline=1)
-        self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
-        self.ha.fetch_node_status = get_node_status(wal_position=1)
-        self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5})
-        self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
-        self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
-        # manual failover from the previous leader to us won't happen if we hold the nofailover flag
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, None))
-        self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
 
-        # Failover scheduled time must include timezone
+        # other members with failover_limitation_s
-        scheduled = datetime.datetime.now()
+        with patch('patroni.ha.logger.info') as mock_info:
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
+            self.ha.fetch_node_status = get_node_status(nofailover=True)
-        self.ha.run_cycle()
+            self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
+            self.assertEqual(mock_info.call_args_list[0][0], ('Member %s is %s', 'leader', 'not allowed to promote'))
+        with patch('patroni.ha.logger.info') as mock_info:
+            self.ha.fetch_node_status = get_node_status(watchdog_failed=True)
+            self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
+            self.assertEqual(mock_info.call_args_list[0][0], ('Member %s is %s', 'leader', 'not watchdog capable'))
+        with patch('patroni.ha.logger.info') as mock_info:
+            self.ha.fetch_node_status = get_node_status(timeline=1)
+            self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
+            self.assertEqual(mock_info.call_args_list[0][0],
+                             ('Timeline %s of member %s is behind the cluster timeline %s', 1, 'leader', 2))
+        with patch('patroni.ha.logger.info') as mock_info:
+            self.ha.fetch_node_status = get_node_status(wal_position=1)
+            self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5})
+            self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
+            self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
+            self.assertEqual(mock_info.call_args_list[0][0], ('Member %s exceeds maximum replication lag', 'leader'))
 
+    @patch('patroni.postgresql.citus.CitusHandler.is_coordinator', Mock(return_value=False))
+    def test_scheduled_switchover_from_leader(self):
+        self.ha.has_lock = true  # I am the leader
+
+        self.ha.fetch_node_status = get_node_status()
+
+        # switchover scheduled time must include timezone
+        with patch('patroni.ha.logger.warning') as mock_warning:
+            scheduled = datetime.datetime.now()
+            self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'blabla', scheduled))
+            self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
+            self.assertIn('Incorrect value of scheduled_at: %s', mock_warning.call_args_list[0][0])
+
+        # scheduled now
         scheduled = datetime.datetime.utcnow().replace(tzinfo=tzutc)
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'b', scheduled))
-        self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
+        self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
+        self.assertEqual('switchover: demoting myself', self.ha.run_cycle())
 
-        scheduled = scheduled + datetime.timedelta(seconds=30)
+        # scheduled in the future
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
+        with patch('patroni.ha.logger.info') as mock_info:
-        self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
+            scheduled = scheduled + datetime.timedelta(seconds=30)
+            self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'blabla', scheduled))
+            self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
+            self.assertIn('Awaiting %s at %s (in %.0f seconds)', mock_info.call_args_list[0][0])
 
-        scheduled = scheduled + datetime.timedelta(seconds=-600)
+        # stale value
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
+        with patch('patroni.ha.logger.warning') as mock_warning:
-        self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
+            scheduled = scheduled + datetime.timedelta(seconds=-600)
+            self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'b', scheduled))
+            self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
+            self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
+            self.assertIn('Found a stale %s value, cleaning up: %s', mock_warning.call_args_list[0][0])
 
-        scheduled = None
+    def test_manual_switchover_from_leader_in_pause(self):
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
+        self.ha.has_lock = true  # I am the leader
-        self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
+        self.ha.is_paused = true
+
+        # no candidate
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, '', None))
+        with patch('patroni.ha.logger.warning') as mock_warning:
+            self.assertEqual('PAUSE: no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
+            mock_warning.assert_called_with(
+                '%s is possible only to a specific candidate in a paused state', 'Switchover')
 
     def test_manual_failover_from_leader_in_pause(self):
         self.ha.has_lock = true
+        self.ha.fetch_node_status = get_node_status()
         self.ha.is_paused = true
-        scheduled = datetime.datetime.now()
+
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
+        # failover from me, candidate is healthy
-        self.assertEqual('PAUSE: no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, None, 'b', None))
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, '', None))
+        self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
-        self.assertEqual('PAUSE: no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
+        self.assertEqual('PAUSE: manual failover: demoting myself', self.ha.run_cycle())
+        self.ha.cluster.members.pop()
 
     def test_manual_failover_from_leader_in_synchronous_mode(self):
-        self.p.is_leader = true
-        self.ha.has_lock = true
         self.ha.is_synchronous_mode = true
-        self.ha.is_failover_possible = false
         self.ha.process_sync_replication = Mock()
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None), (self.p.name, None))
+        self.ha.fetch_node_status = get_node_status()
-        self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
+
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None), (self.p.name, 'a'))
+        # I am the leader
-        self.ha.is_failover_possible = true
+        self.p.is_primary = true
+        self.ha.has_lock = true
+
+        # the candidate is not in sync members but we allow failover to an async candidate
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, None, 'b', None), sync=(self.p.name, 'a'))
+        self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
         self.assertEqual('manual failover: demoting myself', self.ha.run_cycle())
+        self.ha.cluster.members.pop()
+
+    def test_manual_switchover_from_leader_in_synchronous_mode(self):
+        self.ha.is_synchronous_mode = true
+        self.ha.process_sync_replication = Mock()
+
+        # I am the leader
+        self.p.is_primary = true
+        self.ha.has_lock = true
+
+        # candidate specified is not in sync members
+        with patch('patroni.ha.logger.warning') as mock_warning:
+            self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None),
+                                                                  sync=(self.p.name, 'blabla'))
+            self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
+            self.assertEqual(mock_warning.call_args_list[0][0],
+                             ('%s candidate=%s does not match with sync_standbys=%s', 'Switchover', 'a', 'blabla'))
+
+        # the candidate is in sync members and is healthy
+        self.ha.fetch_node_status = get_node_status(wal_position=305419896)
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None),
+                                                              sync=(self.p.name, 'a'))
+        self.ha.cluster.members.append(Member(0, 'a', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
+        self.assertEqual('switchover: demoting myself', self.ha.run_cycle())
+
+        # the candidate is in sync members but is not healthy
+        with patch('patroni.ha.logger.info') as mock_info:
+            self.ha.fetch_node_status = get_node_status(nofailover=true)
+            self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
+            self.assertEqual(mock_info.call_args_list[0][0], ('Member %s is %s', 'a', 'not allowed to promote'))
 
     def test_manual_failover_process_no_leader(self):
-        self.p.is_leader = false
+        self.p.is_primary = false
-        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', self.p.name, None))
-        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None))
         self.p.set_role('replica')
-        self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
+
-        self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
+        # failover to another member, fetch_node_status for candidate fails
-        self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
+        with patch('patroni.ha.logger.warning') as mock_warning:
-        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, self.p.name, '', None))
+            self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None))
-        self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
+            self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
-        self.ha.fetch_node_status = get_node_status(reachable=False)  # inaccessible, in_recovery
+            self.assertEqual(mock_warning.call_args_list[1][0],
+                             ('%s: member %s is %s', 'manual failover', 'leader', 'not reachable'))
+
+        # failover to another member, candidate is accessible, in_recovery
         self.p.set_role('replica')
-        self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
+        self.ha.fetch_node_status = get_node_status()
-        # set failover flag to True for all members of the cluster
+        self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
+
+        # set nofailover flag to True for all members of the cluster
         # this should elect the current member, as we are not going to call the API for it.
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
-        self.ha.fetch_node_status = get_node_status(nofailover=True)  # accessible, in_recovery
+        self.ha.fetch_node_status = get_node_status(nofailover=True)
-        self.p.set_role('replica')
         self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
-        # same as previous, but set the current member to nofailover. In no case it should be elected as a leader
+
+        # failover to me but I am set to nofailover. In no case I should be elected as a leader
+        self.p.set_role('replica')
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'postgresql0', None))
         self.ha.patroni.nofailover = True
         self.assertEqual(self.ha.run_cycle(), 'following a different leader because I am not allowed to promote')
 
+        self.ha.patroni.nofailover = False
+
+        # failover to another member that is on an older timeline (only failover_limitation() is checked)
+        with patch('patroni.ha.logger.info') as mock_info:
+            self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'b', None))
+            self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
+            self.ha.fetch_node_status = get_node_status(timeline=1)
+            self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
+            mock_info.assert_called_with('%s: to %s, i am %s', 'manual failover', 'b', 'postgresql0')
+
+        # failover to another member lagging behind the cluster_lsn (only failover_limitation() is checked)
+        with patch('patroni.ha.logger.info') as mock_info:
+            self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5})
+            self.ha.fetch_node_status = get_node_status(wal_position=1)
+            self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
+            mock_info.assert_called_with('%s: to %s, i am %s', 'manual failover', 'b', 'postgresql0')
+
+    def test_manual_switchover_process_no_leader(self):
+        self.p.is_primary = false
+        self.p.set_role('replica')
+
+        # I was the leader, other members are healthy
+        self.ha.fetch_node_status = get_node_status()
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, self.p.name, '', None))
+        self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
+
+        # I was the leader, I am the only healthy member
+        with patch('patroni.ha.logger.info') as mock_info:
+            self.ha.fetch_node_status = get_node_status(reachable=False)  # inaccessible, in_recovery
+            self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
+            self.assertEqual(mock_info.call_args_list[0][0], ('Member %s is %s', 'leader', 'not reachable'))
+            self.assertEqual(mock_info.call_args_list[1][0], ('Member %s is %s', 'other', 'not reachable'))
+
     def test_manual_failover_process_no_leader_in_synchronous_mode(self):
         self.ha.is_synchronous_mode = true
-        self.p.is_leader = false
+        self.p.is_primary = false
+        self.ha.fetch_node_status = get_node_status(nofailover=True)  # other nodes are not healthy
 
-        # switchover to a specific node, which name doesn't match our name (postgresql0)
+        # manual failover when our name (postgresql0) isn't in the /sync key and the candidate node is not available
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None),
+                                                                 sync=('leader1', 'blabla'))
+        self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
+
+        # manual failover when the candidate node isn't available but our name is in the /sync key
+        # while other sync node is nofailover
+        with patch('patroni.ha.logger.warning') as mock_warning:
+            self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None),
+                                                                     sync=('leader1', 'postgresql0'))
+            self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(), CaseInsensitiveSet()))
+            self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
+            self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
+            self.assertEqual(mock_warning.call_args_list[0][0],
+                             ('%s: member %s is %s', 'manual failover', 'other', 'not allowed to promote'))
+
+        # manual failover to our node (postgresql0),
+        # which name is not in sync nodes list (some sync nodes are available)
+        self.p.set_role('replica')
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'postgresql0', None),
+                                                                 sync=('leader1', 'other'))
+        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(['leader1']),
+                                                               CaseInsensitiveSet(['leader1'])))
+        self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
+
+    def test_manual_switchover_process_no_leader_in_synchronous_mode(self):
+        self.ha.is_synchronous_mode = true
+        self.p.is_primary = false
+
+        # to a specific node, which name doesn't match our name (postgresql0)
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'other', None))
         self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
 
-        # switchover to our node (postgresql0), which name is not in sync nodes list
+        # to our node (postgresql0), which name is not in sync nodes list
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'postgresql0', None),
                                                                  sync=('leader1', 'blabla'))
         self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
 
-        # switchover from a specific leader, but our name (postgresql0) is not in the sync nodes list
+        # without candidate, our name (postgresql0) is not in the sync nodes list
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', '', None),
                                                                  sync=('leader', 'blabla'))
         self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
@@ -794,53 +981,39 @@ class TestHa(PostgresInit):
                                                                  sync=('postgresql0'))
         self.ha.patroni.nofailover = True
         self.assertEqual(self.ha.run_cycle(), 'following a different leader because I am not allowed to promote')
-        self.ha.patroni.nofailover = False
-
-        # manual failover when our name (postgresql0) isn't in the /sync key and the `other` node is not available
-        self.ha.fetch_node_status = get_node_status(nofailover=True)  # accessible, in_recovery
-        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None),
-                                                                 sync=('leader1', 'blabla'))
-        self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
-
-        # manual failover when the `other` node isn't available but our name is in the /sync key
-        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None),
-                                                                 sync=('leader1', 'postgresql0'))
-        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(), CaseInsensitiveSet()))
-        self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
-        self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
-
-        # manual failover to our node (postgresql0),
-        # which name is not in sync nodes list (the leader and all sync nodes are not available)
-        self.p.set_role('replica')
-        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'postgresql0', None),
-                                                                 sync=('leader1', 'other'))
-        self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
-
-        # manual failover to our node (postgresql0),
-        # which name is not in sync nodes list (some sync nodes are available)
-        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'postgresql0', None),
-                                                                 sync=('leader1', 'other'))
-        self.p.set_role('replica')
-        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(['leader1']),
-                                                               CaseInsensitiveSet(['leader1'])))
-        self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
 
     def test_manual_failover_process_no_leader_in_pause(self):
         self.ha.is_paused = true
+
+        # I am running as primary, cluster is unlocked, the candidate is allowed to promote
+        # but we are in pause
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
         self.assertEqual(self.ha.run_cycle(), 'PAUSE: continue to run as primary without lock')
+
+    def test_manual_switchover_process_no_leader_in_pause(self):
+        self.ha.is_paused = true
+
+        # I am running as primary, cluster is unlocked, no candidate specified
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', '', None))
         self.assertEqual(self.ha.run_cycle(), 'PAUSE: continue to run as primary without lock')
-        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'blabla', None))
+
-        self.assertEqual('PAUSE: acquired session lock as a leader', self.ha.run_cycle())
+        # the candidate is not running
-        self.p.is_leader = false
+        with patch('patroni.ha.logger.warning') as mock_warning:
+            self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'blabla', None))
+            self.assertEqual('PAUSE: acquired session lock as a leader', self.ha.run_cycle())
+            self.assertEqual(
+                mock_warning.call_args_list[0][0],
+                ('%s: removing failover key because failover candidate is not running', 'switchover'))
+
+        # switchover to me, I am not leader
+        self.p.is_primary = false
         self.p.set_role('replica')
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', self.p.name, None))
         self.assertEqual(self.ha.run_cycle(), 'PAUSE: promoted self to leader by acquiring session lock')
 
     def test_is_healthiest_node(self):
         self.ha.is_failsafe_mode = true
-        self.ha.state_handler.is_leader = false
+        self.ha.state_handler.is_primary = false
         self.ha.patroni.nofailover = False
         self.ha.fetch_node_status = get_node_status()
         self.ha.dcs._last_failsafe = {'foo': ''}
@@ -854,7 +1027,7 @@ class TestHa(PostgresInit):
         self.assertFalse(self.ha.is_healthiest_node())
 
     def test__is_healthiest_node(self):
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.ha.cluster = get_cluster_initialized_without_leader(sync=('postgresql1', self.p.name))
         self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
         self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
@@ -883,7 +1056,10 @@ class TestHa(PostgresInit):
         member = Member(0, 'test', 1, {'api_url': 'http://127.0.0.1:8011/patroni'})
         self.ha.fetch_node_status(member)
         member = Member(0, 'test', 1, {'api_url': 'http://localhost:8011/patroni'})
-        self.ha.fetch_node_status(member)
+        self.ha.patroni.request = Mock()
+        self.ha.patroni.request.return_value.data = b'{"wal":{"location":1},"role":"primary"}'
+        ret = self.ha.fetch_node_status(member)
+        self.assertFalse(ret.in_recovery)
 
     @patch.object(Rewind, 'pg_rewind', true)
     @patch.object(Rewind, 'check_leader_is_not_in_recovery', true)
@@ -950,7 +1126,7 @@ class TestHa(PostgresInit):
         self.assertTrue(self.ha.restart_matches("replica", "9.5.2", False))
 
     def test_process_healthy_cluster_in_pause(self):
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.ha.is_paused = true
         self.p.name = 'leader'
         self.ha.cluster = get_cluster_initialized_with_leader()
@@ -961,7 +1137,7 @@ class TestHa(PostgresInit):
     @patch('patroni.postgresql.mtime', Mock(return_value=1588316884))
     @patch('builtins.open', mock_open(read_data='1\t0/40159C0\tno recovery target specified\n'))
     def test_process_healthy_standby_cluster_as_standby_leader(self):
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.p.name = 'leader'
         self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
         self.p.config.check_recovery_conf = Mock(return_value=(False, False))
@@ -973,7 +1149,7 @@ class TestHa(PostgresInit):
         self.assertEqual(self.ha.run_cycle(), 'promoted self to a standby leader because i had the session lock')
 
     def test_process_healthy_standby_cluster_as_cascade_replica(self):
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.p.name = 'replica'
         self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
         self.assertEqual(self.ha.run_cycle(),
@@ -983,7 +1159,7 @@ class TestHa(PostgresInit):
 
     @patch.object(Cluster, 'is_unlocked', Mock(return_value=True))
     def test_process_unhealthy_standby_cluster_as_standby_leader(self):
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.p.name = 'leader'
         self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
         self.ha.sysid_valid = true
@@ -993,13 +1169,13 @@ class TestHa(PostgresInit):
     @patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
     @patch.object(Rewind, 'can_rewind', PropertyMock(return_value=True))
     def test_process_unhealthy_standby_cluster_as_cascade_replica(self):
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.p.name = 'replica'
         self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
         self.assertTrue(self.ha.run_cycle().startswith('running pg_rewind from remote_member:'))
 
     def test_recover_unhealthy_leader_in_standby_cluster(self):
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.p.name = 'leader'
         self.p.is_running = false
         self.p.follow = false
@@ -1008,7 +1184,7 @@ class TestHa(PostgresInit):
 
     @patch.object(Cluster, 'is_unlocked', Mock(return_value=True))
     def test_recover_unhealthy_unlocked_standby_cluster(self):
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.p.name = 'leader'
         self.p.is_running = false
         self.p.follow = false
@@ -1068,7 +1244,7 @@ class TestHa(PostgresInit):
         check_calls([(update_lock, True), (demote, True)])
 
         self.ha.has_lock = false
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.assertEqual(self.ha.run_cycle(),
                          'no action. I am (postgresql0), a secondary, and following a leader (leader)')
         check_calls([(update_lock, False), (demote, False)])
@@ -1079,7 +1255,7 @@ class TestHa(PostgresInit):
         f = Failover(0, self.p.name, '', None)
         self.ha.cluster = get_cluster_initialized_with_leader(f)
         self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
-        self.assertEqual(self.ha.run_cycle(), 'manual failover: demoting myself')
+        self.assertEqual(self.ha.run_cycle(), 'switchover: demoting myself')
 
     @patch('patroni.ha.Ha.demote')
     def test_failover_immediately_on_zero_primary_start_timeout(self, demote):
@@ -1202,7 +1378,7 @@ class TestHa(PostgresInit):
         self.ha.is_synchronous_mode = true
 
         mock_set_sync = self.p.sync_handler.set_synchronous_standby_names = Mock()
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.p.set_role('replica')
         self.ha.has_lock = true
         mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
@@ -1225,7 +1401,7 @@ class TestHa(PostgresInit):
     def test_unhealthy_sync_mode(self):
         self.ha.is_synchronous_mode = true
 
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.p.set_role('replica')
         self.p.name = 'other'
         self.ha.cluster = get_cluster_initialized_without_leader(sync=('leader', 'other2'))
@@ -1256,7 +1432,7 @@ class TestHa(PostgresInit):
         self.ha.is_synchronous_mode = true
 
         self.p.name = 'other'
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.p.set_role('replica')
         mock_restart = self.p.restart = Mock(return_value=True)
         self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
@@ -1287,6 +1463,21 @@ class TestHa(PostgresInit):
         mock_restart.assert_called_once()
         self.ha.dcs.get_cluster.assert_not_called()
 
+    @patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
+    def test_enable_synchronous_mode(self):
+        self.ha.is_synchronous_mode = true
+        self.ha.has_lock = true
+        self.p.name = 'leader'
+        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(), CaseInsensitiveSet()))
+        self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
+        with patch('patroni.ha.logger.info') as mock_logger:
+            self.ha.run_cycle()
+            self.assertEqual(mock_logger.call_args_list[0][0][0], 'Enabled synchronous replication')
+        self.ha.dcs.write_sync_state = Mock(return_value=None)
+        with patch('patroni.ha.logger.warning') as mock_logger:
+            self.ha.run_cycle()
+            self.assertEqual(mock_logger.call_args[0][0], 'Updating sync state failed')
+
     def test_effective_tags(self):
         self.ha._disable_sync = True
         self.assertEqual(self.ha.get_effective_tags(), {'foo': 'bar', 'nosync': True})
@@ -1359,7 +1550,7 @@ class TestHa(PostgresInit):
     @patch('sys.exit', return_value=1)
     def test_abort_join(self, exit_mock):
         self.ha.cluster = get_cluster_not_initialized_without_leader()
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.ha.run_cycle()
         exit_mock.assert_called_once_with(1)
 
@@ -1410,6 +1601,7 @@ class TestHa(PostgresInit):
     @patch('os.open', Mock())
     @patch('os.fsync', Mock())
     @patch('os.close', Mock())
+    @patch('os.chmod', Mock())
     @patch('os.rename', Mock())
     @patch('patroni.postgresql.Postgresql.is_starting', Mock(return_value=False))
     @patch('builtins.open', mock_open())
@@ -1418,7 +1610,7 @@ class TestHa(PostgresInit):
     @patch.object(SlotsHandler, 'sync_replication_slots', Mock(return_value=['ls']))
     def test_follow_copy(self):
         self.ha.cluster.config.data['slots'] = {'ls': {'database': 'a', 'plugin': 'b'}}
-        self.p.is_leader = false
+        self.p.is_primary = false
         self.assertTrue(self.ha.run_cycle().startswith('Copying logical slots'))
 
     def test_acquire_lock(self):
@@ -1438,3 +1630,13 @@ class TestHa(PostgresInit):
             self.assertEqual(self.ha.patroni.request.call_args[1]['timeout'], 2)
             mock_logger.assert_called()
             self.assertTrue(mock_logger.call_args[0][0].startswith('Request to Citus coordinator'))
+
+    def test_has_members_eligible_to_promote(self):
+        self.ha.fetch_node_status = get_node_status()
+        members = [
+            Member(0, 'test', 1, {'api_url': 'http://127.0.0.1:8011/patroni', 'conn_url': 'postgres://127.0.0.1:5432/postgres'}),
+            Member(0, 'test2', 1, {'api_url': 'http://127.0.0.1:8011/patroni', 'conn_url': 'postgres://127.0.0.1:5432/postgres'}),
+        ]
+        with patch('patroni.ha.logger.info') as mock_logger:
+            self.assertTrue(self.ha.has_members_eligible_to_promote(members, fast_path=True))
+            mock_logger.assert_not_called()

tests/test_kubernetes.py

@@ -8,8 +8,8 @@ import unittest
 import urllib3
 
 from mock import Mock, PropertyMock, mock_open, patch
-from patroni.dcs.kubernetes import Cluster, k8s_client, k8s_config, K8sConfig, K8sConnectionFailed,\
+from patroni.dcs.kubernetes import Cluster, k8s_client, k8s_config, K8sConfig, K8sConnectionFailed, \
-    K8sException, K8sObject, Kubernetes, KubernetesError, KubernetesRetriableException,\
+    K8sException, K8sObject, Kubernetes, KubernetesError, KubernetesRetriableException, \
     Retry, RetryFailedError, SERVICE_HOST_ENV_NAME, SERVICE_PORT_ENV_NAME
 from threading import Thread
 from . import MockResponse, SleepException
@@ -86,8 +86,8 @@ class TestK8sConfig(unittest.TestCase):
             with patch('os.environ', env):
                 self.assertRaises(k8s_config.ConfigException, k8s_config.load_incluster_config)
 
-        with patch('os.environ', {SERVICE_HOST_ENV_NAME: 'a', SERVICE_PORT_ENV_NAME: '1'}),\
+        with patch('os.environ', {SERVICE_HOST_ENV_NAME: 'a', SERVICE_PORT_ENV_NAME: '1'}), \
-                patch('os.path.isfile', Mock(side_effect=[False, True, True, False, True, True, True, True])),\
+                patch('os.path.isfile', Mock(side_effect=[False, True, True, False, True, True, True, True])), \
                 patch('builtins.open', Mock(side_effect=[
                     mock_open()(), mock_open(read_data='a')(), mock_open(read_data='a')(),
                     mock_open()(), mock_open(read_data='a')(), mock_open(read_data='a')()])):
@@ -98,8 +98,8 @@ class TestK8sConfig(unittest.TestCase):
             self.assertEqual(k8s_config.headers.get('authorization'), 'Bearer a')
 
     def test_refresh_token(self):
-        with patch('os.environ', {SERVICE_HOST_ENV_NAME: 'a', SERVICE_PORT_ENV_NAME: '1'}),\
+        with patch('os.environ', {SERVICE_HOST_ENV_NAME: 'a', SERVICE_PORT_ENV_NAME: '1'}), \
-                patch('os.path.isfile', Mock(side_effect=[True, True, False, True, True, True])),\
+                patch('os.path.isfile', Mock(side_effect=[True, True, False, True, True, True])), \
                 patch('builtins.open', Mock(side_effect=[
                     mock_open(read_data='cert')(), mock_open(read_data='a')(),
                     mock_open()(), mock_open(read_data='b')(), mock_open(read_data='c')()])):
@@ -138,10 +138,10 @@ class TestK8sConfig(unittest.TestCase):
 
         config["users"][0]["user"]["client-key-data"] = base64.b64encode(b'foobar').decode('utf-8')
         config["clusters"][0]["cluster"]["certificate-authority-data"] = base64.b64encode(b'foobar').decode('utf-8')
-        with patch('builtins.open', mock_open(read_data=json.dumps(config))),\
+        with patch('builtins.open', mock_open(read_data=json.dumps(config))), \
-                patch('os.write', Mock()), patch('os.close', Mock()),\
+                patch('os.write', Mock()), patch('os.close', Mock()), \
-                patch('os.remove') as mock_remove,\
+                patch('os.remove') as mock_remove, \
-                patch('atexit.register') as mock_atexit,\
+                patch('atexit.register') as mock_atexit, \
                 patch('tempfile.mkstemp') as mock_mkstemp:
             mock_mkstemp.side_effect = [(3, '1.tmp'), (4, '2.tmp')]
             k8s_config.load_kube_config()
@@ -298,11 +298,35 @@ class TestKubernetesConfigMaps(BaseTestKubernetes):
         self.k.touch_member({'state': 'running', 'role': 'replica'})
         self.k.touch_member({'state': 'stopped', 'role': 'primary'})
 
+        self.k._role_label = 'isMaster'
+        self.k._leader_label_value = 'true'
+        self.k._follower_label_value = 'false'
+        self.k._standby_leader_label_value = 'false'
+        self.k._tmp_role_label = 'tmp_role'
+
+        self.k.touch_member({'state': 'running', 'role': 'replica'})
+        mock_patch_namespaced_pod.assert_called()
+        self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['isMaster'], 'false')
+        self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['tmp_role'], 'replica')
+        mock_patch_namespaced_pod.rest_mock()
+
+        self.k._name = 'p-0'
+        self.k.touch_member({'role': 'standby_leader'})
+        mock_patch_namespaced_pod.assert_called()
+        self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['isMaster'], 'false')
+        self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['tmp_role'], 'master')
+        mock_patch_namespaced_pod.rest_mock()
+
+        self.k.touch_member({'role': 'primary'})
+        mock_patch_namespaced_pod.assert_called()
+        self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['isMaster'], 'true')
+        self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['tmp_role'], 'master')
+
     def test_initialize(self):
         self.k.initialize()
 
     def test_delete_leader(self):
-        self.k.delete_leader(1)
+        self.k.delete_leader(self.k.get_cluster().leader, 1)
 
     def test_cancel_initialization(self):
         self.k.cancel_initialization()
@@ -412,6 +436,10 @@ class TestKubernetesEndpoints(BaseTestKubernetes):
         mock_logger_exception.assert_called_once()
         self.assertEqual(('create_config_service failed',), mock_logger_exception.call_args[0])
 
+    @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_endpoints', mock_namespaced_kind, create=True)
+    def test_write_leader_optime(self):
+        self.k.write_leader_optime(12345)
+
 
 def mock_watch(*args):
     return urllib3.HTTPResponse()

tests/test_log.py

@@ -43,7 +43,7 @@ class TestPatroniLogger(unittest.TestCase):
         _LOG.exception('test')
         logger.start()
 
-        with patch.object(logging.Handler, 'format', Mock(side_effect=Exception)),\
+        with patch.object(logging.Handler, 'format', Mock(side_effect=Exception)), \
                 patch('_pytest.logging.LogCaptureHandler.emit', Mock()):
             logging.error('test')
 

tests/test_patroni.py

@@ -40,6 +40,7 @@ class MockFrozenImporter(object):
 @patch('time.sleep', Mock())
 @patch('subprocess.call', Mock(return_value=0))
 @patch('patroni.psycopg.connect', psycopg_connect)
+@patch('urllib3.PoolManager.request', Mock(side_effect=Exception))
 @patch.object(ConfigHandler, 'append_pg_hba', Mock())
 @patch.object(ConfigHandler, 'write_postgresql_conf', Mock())
 @patch.object(ConfigHandler, 'write_recovery_conf', Mock())
@@ -63,6 +64,7 @@ class TestPatroni(unittest.TestCase):
             self.assertRaises(SystemExit, _main)
 
     @patch('pkgutil.iter_importers', Mock(return_value=[MockFrozenImporter()]))
+    @patch('urllib3.PoolManager.request', Mock(side_effect=Exception))
     @patch('sys.frozen', Mock(return_value=True), create=True)
     @patch.object(HTTPServer, '__init__', Mock())
     @patch.object(etcd.Client, 'read', etcd_read)
@@ -183,7 +185,7 @@ class TestPatroni(unittest.TestCase):
 
     def test_reload_config(self):
         self.p.reload_config()
-        self.p.get_tags = Mock(side_effect=Exception)
+        self.p._get_tags = Mock(side_effect=Exception)
         self.p.reload_config(local=True)
 
     def test_nosync(self):

tests/test_postgresql.py

@@ -310,6 +310,17 @@ class TestPostgresql(BaseTestPostgresql):
         self.p.config.write_postgresql_conf()
         self.assertEqual(self.p.config.check_recovery_conf(None), (False, False))
         self.assertEqual(self.p.config.check_recovery_conf(None), (False, False))
+
+        # Config files changed, but can't connect to postgres
+        mock_get_pg_settings.side_effect = PostgresConnectionException('')
+        with patch('patroni.postgresql.config.mtime', mock_mtime):
+            self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
+
+        # Config files didn't change, but postgres crashed or in crash recovery
+        with patch.object(MockPostmaster, 'create_time', Mock(return_value=1234568), create=True):
+            self.assertEqual(self.p.config.check_recovery_conf(None), (False, False))
+
+        # Any other exception raised when executing the query
         mock_get_pg_settings.side_effect = Exception
         with patch('patroni.postgresql.config.mtime', mock_mtime):
             self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
@@ -333,7 +344,7 @@ class TestPostgresql(BaseTestPostgresql):
 
         mock_read_auto = mock_open(read_data=read_data)
         mock_read_auto.return_value.__iter__ = lambda o: iter(o.readline, '')
-        with patch('builtins.open', Mock(side_effect=[mock_open()(), mock_read_auto(), IOError])),\
+        with patch('builtins.open', Mock(side_effect=[mock_open()(), mock_read_auto(), IOError])), \
                 patch('os.chmod', Mock()):
             self.p.config.write_postgresql_conf()
 
@@ -346,8 +357,7 @@ class TestPostgresql(BaseTestPostgresql):
     @patch.object(Postgresql, 'start', Mock())
     def test_follow(self):
         self.p.call_nowait(CallbackAction.ON_START)
-        m = RemoteMember.from_name_and_data('1', {'restore_command': '2', 'primary_slot_name': 'foo',
+        m = RemoteMember('1', {'restore_command': '2', 'primary_slot_name': 'foo', 'conn_kwargs': {'host': 'bar'}})
-                                                  'conn_kwargs': {'host': 'bar'}})
         self.p.follow(m)
         with patch.object(Postgresql, 'ensure_major_version_is_known', Mock(return_value=False)):
             self.assertIsNone(self.p.follow(m))
@@ -364,11 +374,11 @@ class TestPostgresql(BaseTestPostgresql):
         self.assertRaises(psycopg.ProgrammingError, self.p.query, 'blabla')
 
     @patch.object(Postgresql, 'pg_isready', Mock(return_value=STATE_REJECT))
-    def test_is_leader(self):
+    def test_is_primary(self):
-        self.assertTrue(self.p.is_leader())
+        self.assertTrue(self.p.is_primary())
         self.p.reset_cluster_info_state(None)
         with patch.object(Postgresql, '_query', Mock(side_effect=RetryFailedError(''))):
-            self.assertFalse(self.p.is_leader())
+            self.assertFalse(self.p.is_primary())
 
     @patch.object(Postgresql, 'controldata', Mock(return_value={'Database cluster state': 'shut down',
                                                                 'Latest checkpoint location': '0/1ADBC18',
@@ -462,7 +472,7 @@ class TestPostgresql(BaseTestPostgresql):
         self.assertIsNone(self.p.call_nowait(CallbackAction.ON_START))
 
     @patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
-    def test_is_leader_exception(self):
+    def test_is_primary_exception(self):
         self.p.start()
         self.p.query = Mock(side_effect=psycopg.OperationalError("not supported"))
         self.assertTrue(self.p.stop())
@@ -496,8 +506,8 @@ class TestPostgresql(BaseTestPostgresql):
             self.p.remove_data_directory()
         with patch('os.path.isfile', Mock(return_value=True)):
             self.p.remove_data_directory()
-        with patch('os.path.islink', Mock(side_effect=[False, False, True, True])),\
+        with patch('os.path.islink', Mock(side_effect=[False, False, True, True])), \
-                patch('os.listdir', Mock(return_value=['12345'])),\
+                patch('os.listdir', Mock(return_value=['12345'])), \
                 patch('os.path.realpath', Mock(side_effect=['../foo', '../foo_tsp'])):
             self.p.remove_data_directory()
 
@@ -523,8 +533,9 @@ class TestPostgresql(BaseTestPostgresql):
     def test_save_configuration_files(self):
         self.p.config.save_configuration_files()
 
-    @patch('os.path.isfile', Mock(side_effect=[False, True]))
+    @patch('os.path.isfile', Mock(side_effect=[False, True, False, True]))
-    @patch('shutil.copy', Mock(side_effect=IOError))
+    @patch('shutil.copy', Mock(side_effect=[None, IOError]))
+    @patch('os.chmod', Mock())
     def test_restore_configuration_files(self):
         self.p.config.restore_configuration_files()
 
@@ -545,9 +556,7 @@ class TestPostgresql(BaseTestPostgresql):
 
     @patch('time.sleep', Mock())
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
-    @patch.object(MockCursor, 'fetchone')
+    def test_reload_config(self):
-    def test_reload_config(self, mock_fetchone):
-        mock_fetchone.return_value = (1,)
         parameters = self._PARAMETERS.copy()
         parameters.pop('f.oo')
         parameters['wal_buffers'] = '512'
@@ -555,9 +564,14 @@ class TestPostgresql(BaseTestPostgresql):
                   'authentication': {},
                   'retry_timeout': 10, 'listen': '*', 'krbsrvname': 'postgres', 'parameters': parameters}
         self.p.reload_config(config)
-        mock_fetchone.side_effect = Exception
         parameters['b.ar'] = 'bar'
-        self.p.reload_config(config)
+        with patch.object(MockCursor, 'fetchall',
+                          Mock(side_effect=[[('wal_block_size', '8191', None, 'integer', 'internal'),
+                                             ('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
+                                             ('shared_buffers', '16384', '8kB', 'integer', 'postmaster'),
+                                             ('wal_buffers', '-1', '8kB', 'integer', 'postmaster'),
+                                             ('port', '5433', None, 'integer', 'postmaster')], Exception])):
+            self.p.reload_config(config)
         parameters['autovacuum'] = 'on'
         self.p.reload_config(config)
         parameters['autovacuum'] = 'off'
@@ -574,7 +588,10 @@ class TestPostgresql(BaseTestPostgresql):
         self.assertEqual(self.p.config.local_replication_address, {'host': '/tmp', 'port': '5432'})
         self.p.config._server_parameters.pop('unix_socket_directories')
         self.p.config.resolve_connection_addresses()
-        self.assertEqual(self.p.config._local_address, {'port': '5432'})
+        self.assertEqual(self.p.connection_pool.conn_kwargs, {'connect_timeout': 3, 'dbname': 'postgres',
+                                                              'fallback_application_name': 'Patroni',
+                                                              'options': '-c statement_timeout=2000',
+                                                              'password': 'test', 'port': '5432', 'user': 'foo'})
 
     @patch.object(Postgresql, '_version_file_exists', Mock(return_value=True))
     def test_get_major_version(self):
@@ -585,7 +602,7 @@ class TestPostgresql(BaseTestPostgresql):
 
     def test_postmaster_start_time(self):
         now = datetime.datetime.now()
-        with patch.object(MockCursor, "fetchone", Mock(return_value=(now, True, '', '', '', '', False))):
+        with patch.object(MockCursor, "fetchall", Mock(return_value=[(now, True, '', '', '', '', False)])):
             self.assertEqual(self.p.postmaster_start_time(), now.isoformat(sep=' '))
             t = Thread(target=self.p.postmaster_start_time)
             t.start()
@@ -955,3 +972,10 @@ class TestPostgresql2(BaseTestPostgresql):
         gucs = self.p.available_gucs
         self.assertIsInstance(gucs, CaseInsensitiveSet)
         self.assertEqual(gucs, mock_available_gucs.return_value)
+
+    def test_cluster_info_query(self):
+        self.assertIn('diff(pg_catalog.pg_current_wal_flush_lsn(', self.p.cluster_info_query)
+        self.p._major_version = 90600
+        self.assertIn('diff(pg_catalog.pg_current_xlog_flush_location(', self.p.cluster_info_query)
+        self.p._major_version = 90500
+        self.assertIn('diff(pg_catalog.pg_current_xlog_location(', self.p.cluster_info_query)

tests/test_raft.py

@@ -4,7 +4,7 @@ import tempfile
 import time
 
 from mock import Mock, PropertyMock, patch
-from patroni.dcs.raft import Cluster, DynMemberSyncObj, KVStoreTTL,\
+from patroni.dcs.raft import Cluster, DynMemberSyncObj, KVStoreTTL, \
     Raft, RaftError, SyncObjUtility, TCPTransport, _TCPTransport
 from pysyncobj import SyncObjConf, FAIL_REASON
 
@@ -142,25 +142,25 @@ class TestRaft(unittest.TestCase):
         raft._citus_group = '1'
         self.assertTrue(raft.manual_failover('foo', 'bar'))
         raft._citus_group = '0'
+        self.assertTrue(raft.take_leader())
         cluster = raft.get_cluster()
         self.assertIsInstance(cluster, Cluster)
         self.assertIsInstance(cluster.workers[1], Cluster)
+        leader = cluster.leader
+        self.assertTrue(raft.delete_leader(leader))
         self.assertTrue(raft._sync_obj.set(raft.status_path, '{"optime":1234567,"slots":{"ls":12345}}'))
-        leader = raft.get_cluster().leader
+        raft.get_cluster()
         self.assertTrue(raft.update_leader(leader, '1', failsafe={'foo': 'bat'}))
         self.assertTrue(raft._sync_obj.set(raft.failsafe_path, '{"foo"}'))
         self.assertTrue(raft._sync_obj.set(raft.status_path, '{'))
         raft.get_citus_coordinator()
         self.assertTrue(raft.delete_sync_state())
-        self.assertTrue(raft.delete_leader())
         self.assertTrue(raft.set_history_value(''))
         self.assertTrue(raft.delete_cluster())
         raft._citus_group = '1'
         self.assertTrue(raft.delete_cluster())
         raft._citus_group = None
         raft.get_cluster()
-        self.assertTrue(raft.take_leader())
-        raft.get_cluster()
         raft.watch(None, 0.001)
         raft._sync_obj.destroy()
 

tests/test_rewind.py

@@ -65,14 +65,14 @@ class TestRewind(BaseTestPostgresql):
 
     def test_pg_rewind(self):
         r = {'user': '', 'host': '', 'port': '', 'database': '', 'password': ''}
-        with patch.object(Postgresql, 'major_version', PropertyMock(return_value=150000)),\
+        with patch.object(Postgresql, 'major_version', PropertyMock(return_value=150000)), \
                 patch.object(CancellableSubprocess, 'call', Mock(return_value=None)):
             with patch('subprocess.check_output', Mock(return_value=b'boo')):
                 self.assertFalse(self.r.pg_rewind(r))
             with patch('subprocess.check_output', Mock(side_effect=Exception)):
                 self.assertFalse(self.r.pg_rewind(r))
 
-        with patch.object(Postgresql, 'major_version', PropertyMock(return_value=120000)),\
+        with patch.object(Postgresql, 'major_version', PropertyMock(return_value=120000)), \
                 patch('subprocess.check_output', Mock(return_value=b'foo %f %p %r %% % %')):
             with patch.object(CancellableSubprocess, 'call', mock_cancellable_call):
                 self.assertFalse(self.r.pg_rewind(r))
@@ -91,9 +91,10 @@ class TestRewind(BaseTestPostgresql):
                                              'Latest checkpoint location': '0/'})):
             self.r.rewind_or_reinitialize_needed_and_possible(self.leader)
 
-        with patch.object(Postgresql, 'is_running', Mock(return_value=True)),\
+        with patch.object(Postgresql, 'is_running', Mock(return_value=True)), \
-                patch.object(MockCursor, 'fetchone',
+                patch.object(MockCursor, 'fetchone', Mock(side_effect=Exception)), \
-                             Mock(side_effect=[(0, 0, 1, 1, 0, 0, 0, 0, 0, None, None, None), Exception])):
+                patch.object(MockCursor, 'fetchall',
+                             Mock(return_value=[(0, 0, 1, 1, 0, 0, 0, 0, 0, None, None, None)])):
             self.r.rewind_or_reinitialize_needed_and_possible(self.leader)
 
     @patch.object(CancellableSubprocess, 'call', mock_cancellable_call)
@@ -239,7 +240,7 @@ class TestRewind(BaseTestPostgresql):
 
         with patch('os.listdir', Mock(return_value=['000000000000000000000000.ready'])):
             # successful archive_command call
-            with patch.object(CancellableSubprocess, 'call', Mock(return_value=0)):
+            with patch.object(CancellableSubprocess, 'call', Mock(return_value=0)) as mock_subprocess_call:
                 get_guc_value_res = [
                     'on', 'command %f',
                     'always', 'command %f',
@@ -252,6 +253,10 @@ class TestRewind(BaseTestPostgresql):
                                           '000000000000000000000000', 'command 000000000000000000000000'),
                                          mock_logger_info.call_args[0])
                         mock_logger_info.reset_mock()
+                        mock_subprocess_call.assert_called_once()
+                        self.assertEqual(mock_subprocess_call.call_args[0][0], ['command 000000000000000000000000'])
+                        self.assertEqual(mock_subprocess_call.call_args[1]['shell'], True)
+                        mock_subprocess_call.reset_mock()
 
             # failed archive_command call
             with patch.object(CancellableSubprocess, 'call', Mock(return_value=1)):

tests/test_slots.py

@@ -7,6 +7,7 @@ from mock import Mock, PropertyMock, patch
 from threading import Thread
 
 from patroni import psycopg
+from patroni.config import GlobalConfig
 from patroni.dcs import Cluster, ClusterConfig, Member, SyncState
 from patroni.postgresql import Postgresql
 from patroni.postgresql.misc import fsync_dir
@@ -28,11 +29,12 @@ class TestSlotsHandler(BaseTestPostgresql):
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def setUp(self):
         super(TestSlotsHandler, self).setUp()
+        self.p._global_config = GlobalConfig({})
         self.s = self.p.slots_handler
         self.p.start()
-        config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}}}, 1)
+        config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}, 'ls2': None}}, 1)
         self.cluster = Cluster(True, config, self.leader, 0, [self.me, self.other, self.leadermem],
-                               None, SyncState.empty(), None, {'ls': 12345}, None)
+                               None, SyncState.empty(), None, {'ls': 12345, 'ls2': 12345}, None)
 
     def test_sync_replication_slots(self):
         config = ClusterConfig(1, {'slots': {'test_3': {'database': 'a', 'plugin': 'b'},
@@ -43,12 +45,13 @@ class TestSlotsHandler(BaseTestPostgresql):
         with mock.patch('patroni.postgresql.Postgresql._query', Mock(side_effect=psycopg.OperationalError)):
             self.s.sync_replication_slots(cluster, False)
         self.p.set_role('standby_leader')
-        with patch.object(SlotsHandler, 'drop_replication_slot', Mock(return_value=(True, False))),\
+        with patch.object(SlotsHandler, 'drop_replication_slot', Mock(return_value=(True, False))), \
+                patch.object(GlobalConfig, 'is_standby_cluster', PropertyMock(return_value=True)), \
                 patch('patroni.postgresql.slots.logger.debug') as mock_debug:
             self.s.sync_replication_slots(cluster, False)
             mock_debug.assert_called_once()
         self.p.set_role('replica')
-        with patch.object(Postgresql, 'is_leader', Mock(return_value=False)),\
+        with patch.object(Postgresql, 'is_primary', Mock(return_value=False)), \
                 patch.object(SlotsHandler, 'drop_replication_slot') as mock_drop:
             self.s.sync_replication_slots(cluster, False, paused=True)
             mock_drop.assert_not_called()
@@ -67,6 +70,23 @@ class TestSlotsHandler(BaseTestPostgresql):
             with patch.object(Postgresql, 'major_version', PropertyMock(return_value=90618)):
                 self.s.sync_replication_slots(cluster, False)
 
+    def test_cascading_replica_sync_replication_slots(self):
+        """Test sync with a cascading replica so physical slots are present on a replica."""
+        config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}}}, 1)
+        cascading_replica = Member(0, 'test-2', 28, {
+            'state': 'running', 'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
+            'tags': {'replicatefrom': 'postgresql0'}
+        })
+        cluster = Cluster(True, config, self.leader, 0,
+                          [self.me, self.other, self.leadermem, cascading_replica],
+                          None, SyncState.empty(), None, {'ls': 10}, None)
+        self.p.set_role('replica')
+        with patch.object(Postgresql, '_query') as mock_query, \
+                patch.object(Postgresql, 'is_primary', Mock(return_value=False)):
+            mock_query.return_value = [('ls', 'logical', 'b', 'a', 5, 12345, 105)]
+            ret = self.s.sync_replication_slots(cluster, False)
+        self.assertEqual(ret, [])
+
     def test_process_permanent_slots(self):
         config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}},
                                    'ignore_slots': [{'name': 'blabla'}]}, 1)
@@ -76,33 +96,34 @@ class TestSlotsHandler(BaseTestPostgresql):
         self.s.sync_replication_slots(cluster, False)
         with patch.object(Postgresql, '_query') as mock_query:
             self.p.reset_cluster_info_state(None)
-            mock_query.return_value.fetchone.return_value = (
+            mock_query.return_value = [(
                 1, 0, 0, 0, 0, 0, 0, 0, 0, None, None,
                 [{"slot_name": "ls", "type": "logical", "datoid": 5, "plugin": "b",
-                  "confirmed_flush_lsn": 12345, "catalog_xmin": 105}])
+                  "confirmed_flush_lsn": 12345, "catalog_xmin": 105}])]
             self.assertEqual(self.p.slots(), {'ls': 12345})
 
             self.p.reset_cluster_info_state(None)
-            mock_query.return_value.fetchone.return_value = (
+            mock_query.return_value = [(
                 1, 0, 0, 0, 0, 0, 0, 0, 0, None, None,
                 [{"slot_name": "ls", "type": "logical", "datoid": 6, "plugin": "b",
-                  "confirmed_flush_lsn": 12345, "catalog_xmin": 105}])
+                  "confirmed_flush_lsn": 12345, "catalog_xmin": 105}])]
             self.assertEqual(self.p.slots(), {})
 
-    @patch.object(Postgresql, 'is_leader', Mock(return_value=False))
+    @patch.object(Postgresql, 'is_primary', Mock(return_value=False))
     def test__ensure_logical_slots_replica(self):
         self.p.set_role('replica')
         self.cluster.slots['ls'] = 12346
-        with patch.object(SlotsHandler, 'check_logical_slots_readiness', Mock()):
+        with patch.object(SlotsHandler, 'check_logical_slots_readiness', Mock(return_value=False)):
             self.assertEqual(self.s.sync_replication_slots(self.cluster, False), [])
         self.s._schedule_load_slots = False
-        with patch.object(MockCursor, 'execute', Mock(side_effect=psycopg.OperationalError)),\
+        with patch.object(MockCursor, 'execute', Mock(side_effect=psycopg.OperationalError)), \
-                patch.object(SlotsAdvanceThread, 'schedule', Mock(return_value=(True, ['ls']))),\
+                patch.object(SlotsAdvanceThread, 'schedule', Mock(return_value=(True, ['ls']))), \
                 patch.object(psycopg.OperationalError, 'diag') as mock_diag:
             type(mock_diag).sqlstate = PropertyMock(return_value='58P01')
             self.assertEqual(self.s.sync_replication_slots(self.cluster, False), ['ls'])
         self.cluster.slots['ls'] = 'a'
         self.assertEqual(self.s.sync_replication_slots(self.cluster, False), [])
+        self.cluster.config.data['slots']['ls']['database'] = 'b'
         with patch.object(MockCursor, 'rowcount', PropertyMock(return_value=1), create=True):
             self.assertEqual(self.s.sync_replication_slots(self.cluster, False), ['ls'])
 
@@ -116,21 +137,21 @@ class TestSlotsHandler(BaseTestPostgresql):
 
     @patch.object(Postgresql, 'stop', Mock(return_value=True))
     @patch.object(Postgresql, 'start', Mock(return_value=True))
-    @patch.object(Postgresql, 'is_leader', Mock(return_value=False))
+    @patch.object(Postgresql, 'is_primary', Mock(return_value=False))
     def test_check_logical_slots_readiness(self):
         self.s.copy_logical_slots(self.cluster, ['ls'])
-        with patch.object(MockCursor, '__iter__', Mock(return_value=iter([('postgresql0', None)]))),\
+        with patch.object(MockCursor, '__iter__', Mock(return_value=iter([('postgresql0', None)]))), \
-                patch.object(MockCursor, 'fetchone', Mock(side_effect=Exception)):
+                patch.object(MockCursor, 'fetchall', Mock(side_effect=Exception)):
-            self.assertIsNone(self.s.check_logical_slots_readiness(self.cluster, False, None))
+            self.assertFalse(self.s.check_logical_slots_readiness(self.cluster, None))
-        with patch.object(MockCursor, '__iter__', Mock(return_value=iter([('postgresql0', None)]))),\
+        with patch.object(MockCursor, '__iter__', Mock(return_value=iter([('postgresql0', None)]))), \
-                patch.object(MockCursor, 'fetchone', Mock(return_value=(False,))):
+                patch.object(MockCursor, 'fetchall', Mock(return_value=[(False,)])):
-            self.assertIsNone(self.s.check_logical_slots_readiness(self.cluster, False, None))
+            self.assertFalse(self.s.check_logical_slots_readiness(self.cluster, None))
         with patch.object(MockCursor, '__iter__', Mock(return_value=iter([('ls', 100)]))):
-            self.s.check_logical_slots_readiness(self.cluster, False, None)
+            self.s.check_logical_slots_readiness(self.cluster, None)
 
     @patch.object(Postgresql, 'stop', Mock(return_value=True))
     @patch.object(Postgresql, 'start', Mock(return_value=True))
-    @patch.object(Postgresql, 'is_leader', Mock(return_value=False))
+    @patch.object(Postgresql, 'is_primary', Mock(return_value=False))
     def test_on_promote(self):
         self.s.schedule_advance_slots({'foo': {'bar': 100}})
         self.s.copy_logical_slots(self.cluster, ['ls'])
@@ -144,7 +165,7 @@ class TestSlotsHandler(BaseTestPostgresql):
         self.assertRaises(OSError, fsync_dir, 'foo')
 
     def test_slots_advance_thread(self):
-        with patch.object(MockCursor, 'execute', Mock(side_effect=psycopg.OperationalError)),\
+        with patch.object(MockCursor, 'execute', Mock(side_effect=psycopg.OperationalError)), \
                 patch.object(psycopg.OperationalError, 'diag') as mock_diag:
             type(mock_diag).sqlstate = PropertyMock(return_value='58P01')
             self.s.schedule_advance_slots({'foo': {'bar': 100}})

tests/test_validator.py

@@ -15,7 +15,8 @@ config = {
     "scope": "string",
     "restapi": {
         "listen": "127.0.0.2:800",
-        "connect_address": "127.0.0.2:800"
+        "connect_address": "127.0.0.2:800",
+        "verify_client": 'none'
     },
     "bootstrap": {
         "dcs": {

tests/test_zookeeper.py

@@ -7,7 +7,7 @@ from kazoo.handlers.threading import SequentialThreadingHandler
 from kazoo.protocol.states import KeeperState, ZnodeStat
 from kazoo.retry import RetryFailedError
 from mock import Mock, PropertyMock, patch
-from patroni.dcs.zookeeper import Cluster, Leader, PatroniKazooClient,\
+from patroni.dcs.zookeeper import Cluster, Leader, PatroniKazooClient, \
     PatroniSequentialThreadingHandler, ZooKeeper, ZooKeeperError
 
 
@@ -202,7 +202,7 @@ class TestZooKeeper(unittest.TestCase):
         mock_logger.assert_called_once()
 
     def test_delete_leader(self):
-        self.assertTrue(self.zk.delete_leader())
+        self.assertTrue(self.zk.delete_leader(self.zk.get_cluster().leader))
 
     def test_set_failover_value(self):
         self.zk.set_failover_value('')

tox.ini

@@ -77,6 +77,7 @@ platform =
     {[common]platforms}
 allowlist_externals =
     rm
+    true
     {env:OPEN_CMD}
 
 [testenv:dep]
@@ -174,24 +175,52 @@ platform =
     {[common]platforms}
 
 [testenv:docs-{lin,mac,win}]
-description = Build Sphinx documentation
+description = Build Sphinx documentation in HTML format
 labels:
     docs
 deps =
-    sphinx>=4
+    -r requirements.docs.txt
-    sphinx_rtd_theme
+    -r requirements.txt
+    psycopg[binary]
+    psycopg2-binary
 commands =
     sphinx-build \
     -d "{envtmpdir}{/}doctree" docs "{toxworkdir}{/}docs_out" \
     --color -b html \
+    -T -E -W --keep-going \
     {posargs}
 commands_post =
     - {tty:{env:OPEN_CMD} "{toxworkdir}{/}docs_out{/}index.html":true:}
 allowlist_externals =
+    true
     {env:OPEN_CMD}
 platform =
     {[common]platforms}
 
+[testenv:pdf-{lin,mac,win}]
+description = Build Sphinx documentation in PDF format
+labels:
+    docs
+deps =
+    -r requirements.docs.txt
+    -r requirements.txt
+    psycopg[binary]
+    psycopg2-binary
+commands =
+    python -m sphinx -T -E -b latex -d _build/doctrees -D language=en . pdf
+    - latexmk -r pdf/latexmkrc -cd -C pdf/Patroni.tex
+    latexmk -r pdf/latexmkrc -cd -pdf -f -dvi- -ps- -jobname=Patroni -interaction=nonstopmode pdf/Patroni.tex
+
+commands_post =
+    - {tty:{env:OPEN_CMD} "pdf{/}Patroni.pdf":true:}
+allowlist_externals =
+    true
+    latexmk
+    {env:OPEN_CMD}
+platform =
+    {[common]platforms}
+change_dir = docs
+
 [flake8]
 max-line-length = 120
 ignore = D401,W503

typings/urllib3/__init__.pyi

@@ -1,6 +1,7 @@
+from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
 from .poolmanager import PoolManager
 from .response import HTTPResponse
 from .util.request import make_headers
 from .util.timeout import Timeout
 
-__all__ = ['HTTPResponse', 'PoolManager', 'Timeout', 'make_headers']
+__all__ = ['HTTPResponse', 'HTTPConnectionPool', 'HTTPSConnectionPool', 'PoolManager', 'Timeout', 'make_headers']

typings/urllib3/connectionpool.pyi

@@ -0,0 +1,2 @@
+class HTTPConnectionPool: ...
+class HTTPSConnectionPool(HTTPConnectionPool): ...