Release v3.1.2 (#2885)

- bump version
- update release notes

.github/workflows/install_deps.py

@@ -45,8 +45,8 @@ def install_packages(what):
     packages['exhibitor'] = packages['zookeeper']
     packages = packages.get(what, [])
     ver = versions.get(what)
-    if float(ver) >= 15:
+    if float(ver) == 15:
-        packages += ['postgresql-{0}-citus-11.2'.format(ver)]
+        packages += ['postgresql-{0}-citus-12.0'.format(ver)]
     subprocess.call(['sudo', 'apt-get', 'update', '-y'])
     return subprocess.call(['sudo', 'apt-get', 'install', '-y', 'postgresql-' + ver, 'expect-dev'] + packages)
 

.github/workflows/mapping.py

@@ -1 +1 @@
-versions = {'etcd': '9.6', 'etcd3': '14', 'consul': '13', 'exhibitor': '12', 'raft': '11', 'kubernetes': '15'}
+versions = {'etcd': '9.6', 'etcd3': '16', 'consul': '13', 'exhibitor': '12', 'raft': '11', 'kubernetes': '15'}

.github/workflows/tests.yaml

@@ -5,6 +5,7 @@ on:
   push:
     branches:
     - master
+    - 'REL_[0-9]+_[0-9]+'
 
 env:
   CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}
@@ -173,4 +174,28 @@ jobs:
 
     - uses: jakebailey/pyright-action@v1
       with:
-        version: 1.1.320
+        version: 1.1.326
+
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.11
+        cache: pip
+
+    - name: Install dependencies
+      run: pip install tox
+
+    - name: Install package dependencies
+      run: |
+        sudo apt update \
+        && sudo apt install -y \
+            latexmk texlive-latex-extra tex-gyre \
+            --no-install-recommends
+
+    - name: Generate documentation
+      run: tox -m docs

.gitignore

@@ -51,6 +51,7 @@ scm-source.json
 docs/build/
 docs/source/_static/
 docs/source/_templates/
+docs/modules/
 
 # Pycharm IDE
 .idea/

.readthedocs.yaml

@@ -19,3 +19,8 @@ formats:
   - epub
   - pdf
   - htmlzip
+
+python:
+  install:
+    - requirements: requirements.docs.txt
+    - requirements: requirements.txt

README.rst

@@ -12,7 +12,7 @@ Patroni is a template for high availability (HA) PostgreSQL solutions using Pyth
 
 We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely.
 
-Currently supported PostgreSQL versions: 9.3 to 15.
+Currently supported PostgreSQL versions: 9.3 to 16.
 
 **Note to Citus users**: Starting from 3.0 Patroni nicely integrates with the `Citus <https://github.com/citusdata/citus>`__ database extension to Postgres. Please check the `Citus support page <https://github.com/zalando/patroni/blob/master/docs/citus.rst>`__ in the Patroni documentation for more info about how to use Patroni high availability together with a Citus distributed cluster.
 

docs/CONTRIBUTING.rst

@@ -1,182 +1,12 @@
 .. _contributing:
 
-Contributing guidelines
+Contributing
-=======================
+============
 
-Wanna contribute to Patroni? Yay - here is how!
+Resources and information for developers can be found in the pages below.
 
-Chatting
+.. toctree::
---------
+   :maxdepth: 2
 
-Just want to chat with other Patroni users?  Looking for interactive troubleshooting help? Join us on channel `#patroni <https://postgresteam.slack.com/archives/C9XPYG92A>`__ in the `PostgreSQL Slack <https://pgtreats.info/slack-invite>`__.
+   contributing_guidelines
-
+   Patroni API docs<modules/modules>
-Running tests
--------------
-
-Requirements for running behave tests:
-
-1. PostgreSQL packages need to be installed.
-2. PostgreSQL binaries must be available in your `PATH`. You may need to add them to the path with something like `PATH=/usr/lib/postgresql/11/bin:$PATH python -m behave`.
-3. If you'd like to test with external DCSs (e.g., Etcd, Consul, and Zookeeper) you'll need the packages installed and respective services running and accepting unencrypted/unprotected connections on localhost and default port. In the case of Etcd or Consul, the behave test suite could start them up if binaries are available in the `PATH`.
-
-Install dependencies:
-
-.. code-block:: bash
-
-    # You may want to use Virtualenv or specify pip3.
-    pip install -r requirements.txt
-    pip install -r requirements.dev.txt
-
-After you have all dependencies installed, you can run the various test suites:
-
-.. code-block:: bash
-
-    # You may want to use Virtualenv or specify python3.
-
-    # Run flake8 to check syntax and formatting:
-    python setup.py flake8
-
-    # Run the pytest suite in tests/:
-    python setup.py test
-
-    # Run the behave (https://behave.readthedocs.io/en/latest/) test suite in features/;
-    # modify DCS as desired (raft has no dependencies so is the easiest to start with):
-    DCS=raft python -m behave
-
-Testing with tox
-----------------
-
-To run tox tests you only need to install one dependency (other than Python)
-
-.. code-block:: bash
-
-    pip install tox>=4
-
-If you wish to run `behave` tests then you also need docker installed.
-
-Tox configuration in `tox.ini` has "environments" to run the following tasks:
-
-* lint: Python code lint with `flake8`
-* test: unit tests for all available python interpreters with `pytest`,
-  generates XML reports or HTML reports if a TTY is detected
-* dep: detect package dependency conflicts using `pipdeptree`
-* type: static type checking with `pyright`
-* black: code formatting with `black`
-* docker-build: build docker image used for the `behave` env
-* docker-cmd: run arbitrary command with the above image
-* docker-behave-etcd: run tox for behave tests with above image
-* py*behave: run behave with available python interpreters (without docker, although
-  this is what is called inside docker containers)
-* docs: build docs with `sphinx`
-
-Running tox
-^^^^^^^^^^^
-
-To run the default env list; dep, lint, test, and docs, just run:
-
-.. code-block:: bash
-
-    tox
-
-The `test` envs can be run with the label `test`:
-
-.. code-block:: bash
-
-   tox -m test
-
-The `behave` docker tests can be run with the label `behave`:
-
-.. code-block:: bash
-
-   tox -m behave
-
-Similarly, docs has the label `docs`.
-
-All other envs can be run with their respective env names:
-
-.. code-block:: bash
-
-   tox -e lint
-   tox -e py39-test-lin
-
-It is also possible to select partial env lists using `factors`. For example, if you want to run
-all envs for python 3.10:
-
-.. code-block:: bash
-
-    tox -f py310
-
-This is equivalent to running all the envs listed below:
-
-.. code-block:: bash
-
-    $ tox -l -f py310
-    py310-test-lin
-    py310-test-mac
-    py310-test-win
-    py310-type-lin
-    py310-type-mac
-    py310-type-win
-    py310-behave-etcd-lin
-    py310-behave-etcd-win
-    py310-behave-etcd-mac
-
-
-You can list all configured combinations of environments with tox (>=v4) like so
-
-.. code-block:: bash
-
-    tox l
-
-The envs `test` and `docs` will attempt to open the HTML output files
-when the job completes, if tox is run with an active terminal. This
-is intended to be for benefit of the developer running this env locally.
-It will attempt to run `open` on a mac and `xdg-open` on Linux.
-To use a different command set the env var `OPEN_CMD` to the name or path of
-the command. If this step fails it will not fail the run overall.
-If you want to disable this facility set the env var `OPEN_CMD` to the `:` no-op command.
-
-.. code-block:: bash
-
-   OPEN_CMD=: tox -m docs
-
-Behave tests
-^^^^^^^^^^^^
-
-Behave tests with `-m behave` will build docker images based on PG_MAJOR version 11 through 15 and then run all
-behave tests. This can take quite a long time to run so you might want to limit the scope to a select version of
-Postgres or to a specific feature set or steps.
-
-To specify the version of postgres include the full name of the dependent image build env that you want and then the
-behave env name. For instance if you want Postgres 15 use:
-
-.. code-block:: bash
-
-    tox -e pg14-docker-build,pg14-docker-behave-etcd-lin
-
-If on the other hand you want to test a specific feature you can pass positional arguments to behave. This will run
-the watchdog behave feature test scenario with all versions of Postgres.
-
-.. code-block:: bash
-
-    tox -m behave -- features/watchdog.feature
-
-Of course you can combine the two.
-
-Reporting issues
-----------------
-
-If you have a question about patroni or have a problem using it, please read the :ref:`README <readme>` before filing an issue.
-Also double check with the current issues on our `Issues Tracker <https://github.com/zalando/patroni/issues>`__.
-
-Contributing a pull request
----------------------------
-
-1) Submit a comment to the relevant issue or create a new issue describing your proposed change.
-2) Do a fork, develop and test your code changes.
-3) Include documentation
-4) Submit a pull request.
-
-You'll get feedback about your pull request as soon as possible.
-
-Happy Patroni hacking ;-)

docs/ENVIRONMENT.rst

@@ -115,7 +115,7 @@ Kubernetes
 -  **PATRONI\_KUBERNETES\_ROLE\_LABEL**: (optional) name of the label containing role (master or replica or other custom value). Patroni will set this label on the pod it runs in. Default value is ``role``.
 -  **PATRONI\_KUBERNETES\_LEADER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is `master`. Default value is `master`.
 -  **PATRONI\_KUBERNETES\_FOLLOWER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is `replica`. Default value is `replica`.
--  **PATRONI\_KUBERNETES\_STANDBY\_LEADER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is ``standby-leader``. Default value is ``standby-leader``.
+-  **PATRONI\_KUBERNETES\_STANDBY\_LEADER\_LABEL\_VALUE**: (optional) value of the pod label when Postgres role is ``standby_leader``. Default value is ``master``.
 -  **PATRONI\_KUBERNETES\_TMP\_ROLE\_LABEL**: (optional) name of the temporary label containing role (master or replica). Value of this label will always use the default of corresponding role. Set only when necessary.
 -  **PATRONI\_KUBERNETES\_USE\_ENDPOINTS**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
 -  **PATRONI\_KUBERNETES\_POD\_IP**: (optional) IP address of the pod Patroni is running in. This value is required when `PATRONI_KUBERNETES_USE_ENDPOINTS` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.

docs/conf.py

@@ -20,10 +20,15 @@
 import os
 
 import sys
+
 sys.path.insert(0, os.path.abspath('..'))
 
 from patroni.version import __version__
 
+project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+module_dir = os.path.abspath(os.path.join(project_root, 'patroni'))
+excludes = ['tests', 'setup.py', 'conf']
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -33,11 +38,21 @@ from patroni.version import __version__
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['sphinx.ext.intersphinx',
+extensions = [
+    'sphinx.ext.intersphinx',
     'sphinx.ext.todo',
     'sphinx.ext.mathjax',
     'sphinx.ext.ifconfig',
-    'sphinx.ext.viewcode']
+    # 'sphinx.ext.viewcode',
+    'sphinx_github_style',  # Generate "View on GitHub" for source code
+    'sphinxcontrib.apidoc',  # For generating module docs from code
+    'sphinx.ext.autodoc',  # For generating module docs from docstrings
+    'sphinx.ext.napoleon',  # For Google and Numpy formatted docstrings
+]
+apidoc_module_dir = module_dir
+apidoc_output_dir = 'modules'
+apidoc_excluded_paths = excludes
+apidoc_separate_modules = True
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -90,10 +105,10 @@ todo_include_todos = True
 # a list of builtin themes.
 #
 
+html_theme = 'sphinx_rtd_theme'
 on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
 if not on_rtd:  # only import and set the theme if we're building docs locally
     import sphinx_rtd_theme
-    html_theme = 'sphinx_rtd_theme'
     html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
@@ -107,6 +122,34 @@ if not on_rtd:  # only import and set the theme if we're building docs locally
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
+# Replace "source" links with "edit on GitHub" when using rtd theme
+html_context = {
+    'display_github': True,
+    'github_user': 'zalando',
+    'github_repo': 'patroni',
+    'github_version': 'master',
+    'conf_py_path': '/docs/',
+}
+
+# sphinx-github-style options, https://sphinx-github-style.readthedocs.io/en/latest/index.html
+
+# The name of the top-level package.
+top_level = "patroni"
+
+# The blob to link to on GitHub - any of "head", "last_tag", or "{blob}"
+# linkcode_blob = 'head'
+
+# The link to your GitHub repository formatted as https://github.com/user/repo
+# If not provided, will attempt to create the link from the html_context dict
+# linkcode_url = f"https://github.com/{html_context['github_user']}/" \
+#                f"{html_context['github_repo']}/{html_context['github_version']}"
+
+# The text to use for the linkcode link
+# linkcode_link_text: str = "View on GitHub"
+
+# A linkcode_resolve() function to use for resolving the link target
+# linkcode_resolve: types.FunctionType
+
 
 # -- Options for HTMLHelp output ------------------------------------------
 
@@ -165,7 +208,6 @@ texinfo_documents = [
 ]
 
 
-
 # -- Options for Epub output ----------------------------------------------
 
 # Bibliographic Dublin Core info.
@@ -187,10 +229,57 @@ epub_copyright = copyright
 epub_exclude_files = ['search.html']
 
 
-
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {'python': ('https://docs.python.org/', None)}
 
+
+# Remove these pages from index, references, toc trees, etc.
+# If the builder is not 'html' then add the API docs modules index to pages to be removed.
+exclude_from_builder = {
+    'latex': ['modules/modules'],
+    'epub': ['modules/modules'],
+}
+# Internal holding list, anything added here will always be excluded
+_docs_to_remove = []
+
+
+def builder_inited(app):
+    """Run during Sphinx `builder-inited` phase.
+
+    Set a config value to builder name and add module docs to `docs_to_remove`.
+    """
+    print(f'The builder is: {app.builder.name}')
+    app.add_config_value('builder', app.builder.name, 'env')
+
+    # Remove pages when builder matches any referenced in exclude_from_builder
+    if exclude_from_builder.get(app.builder.name):
+        _docs_to_remove.extend(exclude_from_builder[app.builder.name])
+
+
+def env_get_outdated(app, env, added, changed, removed):
+    """Run during Sphinx `env-get-outdated` phase.
+
+    Remove the items listed in `docs_to_remove` from known pages.
+    """
+    added.difference_update(_docs_to_remove)
+    changed.difference_update(_docs_to_remove)
+    removed.update(_docs_to_remove)
+    return []
+
+
+def doctree_read(app, doctree):
+    """Run during Sphinx `doctree-read` phase.
+
+    Remove the items listed in `docs_to_remove` from the table of contents.
+    """
+    from sphinx import addnodes
+    for toc_tree_node in doctree.traverse(addnodes.toctree):
+        for e in toc_tree_node['entries']:
+            ref = str(e[1])
+            if ref in _docs_to_remove:
+                toc_tree_node['entries'].remove(e)
+
+
 # A possibility to have an own stylesheet, to add new rules or override existing ones
 # For the latter case, the CSS specificity of the rules should be higher than the default ones
 def setup(app):
@@ -198,3 +287,8 @@ def setup(app):
         app.add_css_file('custom.css')
     else:
         app.add_stylesheet('custom.css')
+
+    # Run extra steps to remove module docs when running with a non-html builder
+    app.connect('builder-inited', builder_inited)
+    app.connect('env-get-outdated', env_get_outdated)
+    app.connect('doctree-read', doctree_read)

docs/contributing_guidelines.rst

@@ -0,0 +1,182 @@
+.. _contributing_guidelines:
+
+Contributing guidelines
+=======================
+
+Wanna contribute to Patroni? Yay - here is how!
+
+Chatting
+--------
+
+Just want to chat with other Patroni users?  Looking for interactive troubleshooting help? Join us on channel `#patroni <https://postgresteam.slack.com/archives/C9XPYG92A>`__ in the `PostgreSQL Slack <https://pgtreats.info/slack-invite>`__.
+
+Running tests
+-------------
+
+Requirements for running behave tests:
+
+1. PostgreSQL packages need to be installed.
+2. PostgreSQL binaries must be available in your `PATH`. You may need to add them to the path with something like `PATH=/usr/lib/postgresql/11/bin:$PATH python -m behave`.
+3. If you'd like to test with external DCSs (e.g., Etcd, Consul, and Zookeeper) you'll need the packages installed and respective services running and accepting unencrypted/unprotected connections on localhost and default port. In the case of Etcd or Consul, the behave test suite could start them up if binaries are available in the `PATH`.
+
+Install dependencies:
+
+.. code-block:: bash
+
+    # You may want to use Virtualenv or specify pip3.
+    pip install -r requirements.txt
+    pip install -r requirements.dev.txt
+
+After you have all dependencies installed, you can run the various test suites:
+
+.. code-block:: bash
+
+    # You may want to use Virtualenv or specify python3.
+
+    # Run flake8 to check syntax and formatting:
+    python setup.py flake8
+
+    # Run the pytest suite in tests/:
+    python setup.py test
+
+    # Run the behave (https://behave.readthedocs.io/en/latest/) test suite in features/;
+    # modify DCS as desired (raft has no dependencies so is the easiest to start with):
+    DCS=raft python -m behave
+
+Testing with tox
+----------------
+
+To run tox tests you only need to install one dependency (other than Python)
+
+.. code-block:: bash
+
+    pip install tox>=4
+
+If you wish to run `behave` tests then you also need docker installed.
+
+Tox configuration in `tox.ini` has "environments" to run the following tasks:
+
+* lint: Python code lint with `flake8`
+* test: unit tests for all available python interpreters with `pytest`,
+  generates XML reports or HTML reports if a TTY is detected
+* dep: detect package dependency conflicts using `pipdeptree`
+* type: static type checking with `pyright`
+* black: code formatting with `black`
+* docker-build: build docker image used for the `behave` env
+* docker-cmd: run arbitrary command with the above image
+* docker-behave-etcd: run tox for behave tests with above image
+* py*behave: run behave with available python interpreters (without docker, although
+  this is what is called inside docker containers)
+* docs: build docs with `sphinx`
+
+Running tox
+^^^^^^^^^^^
+
+To run the default env list; dep, lint, test, and docs, just run:
+
+.. code-block:: bash
+
+    tox
+
+The `test` envs can be run with the label `test`:
+
+.. code-block:: bash
+
+   tox -m test
+
+The `behave` docker tests can be run with the label `behave`:
+
+.. code-block:: bash
+
+   tox -m behave
+
+Similarly, docs has the label `docs`.
+
+All other envs can be run with their respective env names:
+
+.. code-block:: bash
+
+   tox -e lint
+   tox -e py39-test-lin
+
+It is also possible to select partial env lists using `factors`. For example, if you want to run
+all envs for python 3.10:
+
+.. code-block:: bash
+
+    tox -f py310
+
+This is equivalent to running all the envs listed below:
+
+.. code-block:: bash
+
+    $ tox -l -f py310
+    py310-test-lin
+    py310-test-mac
+    py310-test-win
+    py310-type-lin
+    py310-type-mac
+    py310-type-win
+    py310-behave-etcd-lin
+    py310-behave-etcd-win
+    py310-behave-etcd-mac
+
+
+You can list all configured combinations of environments with tox (>=v4) like so
+
+.. code-block:: bash
+
+    tox l
+
+The envs `test` and `docs` will attempt to open the HTML output files
+when the job completes, if tox is run with an active terminal. This
+is intended to be for benefit of the developer running this env locally.
+It will attempt to run `open` on a mac and `xdg-open` on Linux.
+To use a different command set the env var `OPEN_CMD` to the name or path of
+the command. If this step fails it will not fail the run overall.
+If you want to disable this facility set the env var `OPEN_CMD` to the `:` no-op command.
+
+.. code-block:: bash
+
+   OPEN_CMD=: tox -m docs
+
+Behave tests
+^^^^^^^^^^^^
+
+Behave tests with `-m behave` will build docker images based on PG_MAJOR version 11 through 15 and then run all
+behave tests. This can take quite a long time to run so you might want to limit the scope to a select version of
+Postgres or to a specific feature set or steps.
+
+To specify the version of postgres include the full name of the dependent image build env that you want and then the
+behave env name. For instance if you want Postgres 15 use:
+
+.. code-block:: bash
+
+    tox -e pg14-docker-build,pg14-docker-behave-etcd-lin
+
+If on the other hand you want to test a specific feature you can pass positional arguments to behave. This will run
+the watchdog behave feature test scenario with all versions of Postgres.
+
+.. code-block:: bash
+
+    tox -m behave -- features/watchdog.feature
+
+Of course you can combine the two.
+
+Reporting issues
+----------------
+
+If you have a question about patroni or have a problem using it, please read the :ref:`README <readme>` before filing an issue.
+Also double check with the current issues on our `Issues Tracker <https://github.com/zalando/patroni/issues>`__.
+
+Contributing a pull request
+---------------------------
+
+1) Submit a comment to the relevant issue or create a new issue describing your proposed change.
+2) Do a fork, develop and test your code changes.
+3) Include documentation
+4) Submit a pull request.
+
+You'll get feedback about your pull request as soon as possible.
+
+Happy Patroni hacking ;-)

docs/ha_multi_dc.rst

@@ -12,7 +12,7 @@ In both cases, it is important to be clear about the following concepts:
 - You should run the odd number of etcd, ZooKeeper or Consul nodes: 3 or 5!
 
 Synchronous Replication
-----------------------------
+-----------------------
 
 To have a multi DC cluster that can automatically tolerate a zone drop, a minimum of 3 is required.
 
@@ -27,7 +27,7 @@ Regarding postgres, we must deploy at least 2 nodes, in different DC. Then you h
 This enables sync replication and the primary node will choose one of the nodes as synchronous.
 
 Asynchronous Replication
-----------------------------------
+------------------------
 
 With only two data centers it would be better to have two independent etcd clusters and run Patroni :ref:`standby cluster <standby_cluster>` in the second data center. If the first site is down, you can MANUALLY promote the ``standby_cluster``.
 

docs/index.rst

@@ -10,7 +10,7 @@ Patroni is a template for high availability (HA) PostgreSQL solutions using Pyth
 
 We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely. There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
 
-Currently supported PostgreSQL versions: 9.3 to 15.
+Currently supported PostgreSQL versions: 9.3 to 16.
 
 **Note to Citus users**: Starting from 3.0 Patroni nicely integrates with the `Citus <https://github.com/citusdata/citus>`__ database extension to Postgres. Please check the :ref:`Citus support page <citus>` in the Patroni documentation for more info about how to use Patroni high availability together with a Citus distributed cluster.
 
@@ -40,6 +40,13 @@ Currently supported PostgreSQL versions: 9.3 to 15.
 Indices and tables
 ==================
 
-* :ref:`genindex`
+.. ifconfig:: builder == 'html'
-* :ref:`modindex`
+
-* :ref:`search`
+  * :ref:`genindex`
+  * :ref:`modindex`
+  * :ref:`search`
+
+.. ifconfig:: builder != 'html'
+
+  * :ref:`genindex`
+  * :ref:`search`

docs/releases.rst

@@ -3,6 +3,86 @@
 Release notes
 =============
 
+Version 3.1.2
+-------------
+
+**Bugfixes**
+
+- Fixed bug with ``wal_keep_size`` checks (Alexander Kukushkin)
+
+  The ``wal_keep_size`` is a GUC that normally has a unit and Patroni was failing to cast its value to ``int``. As a result the value of ``bootstrap.dcs`` was not written to the ``/config`` key afterwards.
+
+- Detect and resolve inconsistencies between ``/sync`` key and ``synchronous_standby_names`` (Alexander Kukushkin)
+
+  Normally, Patroni updates ``/sync`` and ``synchronous_standby_names`` in a very specific order, but in case of a bug or when someone manually reset ``synchronous_standby_names``, Patroni was getting into an inconsistent state. As a result it was possible that the failover happens to an asynchronous node.
+
+- Read GUC's values when joining running Postgres (Alexander Kukushkin)
+
+  When restarted in ``pause``, Patroni was discarding the ``synchronous_standby_names`` GUC from the ``postgresql.conf``. To solve it and avoid similar issues, Patroni will read GUC's value if it is joining an already running Postgres.
+
+- Silenced annoying warnings when checking for node uniqueness (Alexander Kukushkin)
+
+  ``WARNING`` messages are produced by ``urllib3`` if Patroni is quickly restarted.
+
+
+Version 3.1.1
+-------------
+
+**Bugfixes**
+
+- Reset failsafe state on promote (ChenChangAo)
+
+  If switchover/failover happened shortly after failsafe mode had been activated, the newly promoted primary was demoting itself after failsafe becomes inactive.
+
+- Silence useless warnings in ``patronictl`` (Alexander Kukushkin)
+
+  If ``patronictl`` uses the same patroni.yaml file as Patroni and can access ``PGDATA`` directory it might have been showing annoying warnings about incorrect values in the global configuration.
+
+- Explicitly enable synchronous mode for a corner case (Alexander Kukushkin)
+
+  Synchronous mode effectively was never activated if there are no replicas streaming from the primary.
+
+- Fixed bug with ``0`` integer values validation (Israel Barth Rubio)
+
+  In most cases, it didn't cause any issues, just warnings.
+
+- Don't return logical slots for standby cluster (Alexander Kukushkin)
+
+  Patroni can't create logical replication slots in the standby cluster, thus they should be ignored if they are defined in the global configuration.
+
+- Avoid showing docstring in ``patronictl --help`` output (Israel Barth Rubio)
+
+  The ``click`` module needs to get a special hint for that.
+
+- Fixed bug with ``kubernetes.standby_leader_label_value`` (Alexander Kukushkin)
+
+  This feature effectively never worked.
+
+- Returned cluster system identifier to the ``patronictl list`` output (Polina Bungina)
+
+  The problem was introduced while implementing the support for Citus, where we need to hide the identifier because it is different for coordinator and all workers.
+
+- Override ``write_leader_optime`` method in Kubernetes implementation (Alexander Kukushkin)
+
+  The method is supposed to write shutdown LSN to the leader Endpoint/ConfigMap when there are no healthy replicas available to become the new primary.
+
+- Don't start stopped postgres in pause (Alexander Kukushkin)
+
+  Due to a race condition, Patroni was falsely assuming that the standby should be restarted because some recovery parameters (``primary_conninfo`` or similar) were changed.
+
+- Fixed bug in ``patronictl query`` command (Israel Barth Rubio)
+
+  It didn't work when only ``-m`` argument was provided or when none of ``-r`` or ``-m`` were provided.
+
+- Properly treat integer parameters that are used in the command line to start postgres (Polina Bungina)
+
+  If values are supplied as strings and not casted to integer it was resulting in an incorrect calculation of ``max_prepared_transactions`` based on ``max_connections`` for Citus clusters.
+
+- Don't rely on ``pg_stat_wal_receiver`` when deciding on ``pg_rewind`` (Alexander Kukushkin)
+
+  It could happen that ``received_tli`` reported by ``pg_stat_wal_recevier`` is ahead of the actual replayed timeline, while the timeline reported by ``DENTIFY_SYSTEM`` via replication connection is always correct.
+
+
 Version 3.1.0
 -------------
 

docs/yaml_configuration.rst

@@ -43,16 +43,23 @@ Bootstrap configuration
       -  **- data-checksums**: Must be enabled when pg_rewind is needed on 9.3.
       -  **- encoding: UTF8**: default encoding for new databases.
       -  **- locale: UTF8**: default locale for new databases.
-   -  **users**: Some additional users which need to be created after initializing new cluster
+   -  **users**: Some additional users which need to be created after initializing new cluster, see :ref:`Bootstrap users configuration <bootstrap_users_configuration>` below.
+   -  **post\_bootstrap** or **post\_init**: An additional script that will be executed after initializing the cluster. The script receives a connection string URL (with the cluster superuser as a user name). The PGPASSFILE variable is set to the location of pgpass file.
 
-      -  **admin**: the name of user
+.. _bootstrap_users_configuration:
+
+Bootstrap users configuration
+=============================
+
+Users which need to be created after initializing the cluster:
+
+-  **admin**: the name of user
 
   -  **password**: (optional) password for the user
   -  **options**: list of options for CREATE USER statement
 
     -  **- createrole**
     -  **- createdb**
-   -  **post\_bootstrap** or **post\_init**: An additional script that will be executed after initializing the cluster. The script receives a connection string URL (with the cluster superuser as a user name). The PGPASSFILE variable is set to the location of pgpass file.
 
 .. _citus_settings:
 
@@ -158,7 +165,7 @@ Kubernetes
 -  **role\_label**: (optional) name of the label containing role (master or replica or other custom value). Patroni will set this label on the pod it runs in. Default value is ``role``.
 -  **leader\_label\_value**: (optional) value of the pod label when Postgres role is ``master``. Default value is ``master``.
 -  **follower\_label\_value**: (optional) value of the pod label when Postgres role is ``replica``. Default value is ``replica``.
--  **standby\_leader\_label\_value**: (optional) value of the pod label when Postgres role is ``standby-leader``. Default value is ``standby-leader``.
+-  **standby\_leader\_label\_value**: (optional) value of the pod label when Postgres role is ``standby_leader``. Default value is ``master``.
 -  **tmp_\role\_label**: (optional) name of the temporary label containing role (master or replica). Value of this label will always use the default of corresponding role. Set only when necessary.
 -  **use\_endpoints**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
 -  **pod\_ip**: (optional) IP address of the pod Patroni is running in. This value is required when `use_endpoints` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.

features/patroni_api.feature

@@ -68,6 +68,7 @@ Scenario: check API requests for the primary-replica pair in the pause mode
 	When I kill postmaster on postgres1
 	And I issue a GET request to http://127.0.0.1:8009/replica
 	Then I receive a response code 503
+	And "members/postgres1" key in DCS has state=stopped after 10 seconds
 	When I run patronictl.py restart batman postgres1 --force
 	Then I receive a response returncode 0
 	Then replication works from postgres0 to postgres1 after 20 seconds
@@ -76,7 +77,7 @@ Scenario: check API requests for the primary-replica pair in the pause mode
 	Then I receive a response code 200
 	And I receive a response state running
 	And I receive a response role replica
-	When I run patronictl.py reinit batman postgres1 --force
+	When I run patronictl.py reinit batman postgres1 --force --wait
 	Then I receive a response returncode 0
 	And I receive a response output "Success: reinitialize for member postgres1"
 	And postgres1 role is the secondary after 30 seconds

patroni/__main__.py

@@ -65,6 +65,8 @@ class Patroni(AbstractPatroniDaemon):
 
     def ensure_unique_name(self) -> None:
         """A helper method to prevent splitbrain from operator naming error."""
+        from urllib.parse import urlparse
+        from urllib3.connection import HTTPConnection
         from patroni.dcs import Member
 
         cluster = self.dcs.get_cluster()
@@ -74,7 +76,10 @@ class Patroni(AbstractPatroniDaemon):
         if not isinstance(member, Member):
             return
         try:
-            _ = self.request(member, endpoint="/liveness")
+            parts = urlparse(member.api_url)
+            if isinstance(parts.hostname, str):
+                connection = HTTPConnection(parts.hostname, port=parts.port or 80, timeout=3)
+                connection.connect()
                 logger.fatal("Can't start; there is already a node named '%s' running", self.config['name'])
                 sys.exit(1)
         except Exception:

patroni/api.py

@@ -49,9 +49,23 @@ def check_access(func: Callable[['RestApiHandler'], None]) -> Callable[..., None
 
     :Example:
 
-        @check_access
+        >>> class FooServer:
-        def do_PUT_foo():
+        ...   def check_access(self, *args, **kwargs):
-            pass
+        ...     print(f'In FooServer: {args[0].__class__.__name__}')
+        ...     return True
+        ...
+
+        >>> class Foo:
+        ...   server = FooServer()
+        ...   @check_access
+        ...   def do_PUT_foo(self):
+        ...      print('In do_PUT_foo')
+
+        >>> f = Foo()
+        >>> f.do_PUT_foo()
+        In FooServer: Foo
+        In do_PUT_foo
+
     """
 
     def wrapper(self: 'RestApiHandler', *args: Any, **kwargs: Any) -> None:
@@ -97,6 +111,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         """Write a response that is composed only of the HTTP status.
 
         The response is written with these values separated by space:
+
             * HTTP protocol version;
             * *status_code*;
             * description of *status_code*.
@@ -165,8 +180,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
             * ``tags``: tags that were set through Patroni configuration merged with dynamically applied tags;
             * ``database_system_identifier``: ``Database system identifier`` from ``pg_controldata`` output;
             * ``pending_restart``: ``True`` if PostgreSQL is pending to be restarted;
-        * ``scheduled_restart``: a dictionary with a single key ``schedule``, which is the timestamp for the scheduled
+            * ``scheduled_restart``: a dictionary with a single key ``schedule``, which is the timestamp for the
-            restart;
+                scheduled restart;
             * ``watchdog_failed``: ``True`` if watchdog device is unhealthy;
             * ``logger_queue_size``: log queue length if it is longer than expected;
             * ``logger_records_lost``: number of log records that have been lost while the log queue was full.
@@ -204,32 +219,54 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Is used for handling all health-checks requests. E.g. "GET /(primary|replica|sync|async|etc...)".
 
         The (optional) query parameters and the HTTP response status depend on the requested path:
+
             * ``/``, ``primary``, or ``read-write``:
+
                 * HTTP status ``200``: if a primary with the leader lock.
+
             * ``/standby-leader``:
+
                 * HTTP status ``200``: if holds the leader lock in a standby cluster.
+
             * ``/leader``:
+
                 * HTTP status ``200``: if holds the leader lock.
+
             * ``/replica``:
+
                 * Query parameters:
+
                     * ``lag``: only accept replication lag up to ``lag``. Accepts either an :class:`int`, which
                         represents lag in bytes, or a :class:`str` representing lag in human-readable format (e.g.
                         ``10MB``).
                     * Any custom parameter: will attempt to match them against node tags.
+
                 * HTTP status ``200``: if up and running as a standby and without ``noloadbalance`` tag.
+
             * ``/read-only``:
+
                 * HTTP status ``200``: if up and running and without ``noloadbalance`` tag.
+
             * ``/synchronous`` or ``/sync``:
+
                 * HTTP status ``200``: if up and running as a synchronous standby.
+
             * ``/read-only-sync``:
+
                 * HTTP status ``200``: if up and running as a synchronous standby or primary.
+
             * ``/asynchronous``:
+
                 * Query parameters:
+
                     * ``lag``: only accept replication lag up to ``lag``. Accepts either an :class:`int`, which
                         represents lag in bytes, or a :class:`str` representing lag in human-readable format (e.g.
                         ``10MB``).
+
                 * HTTP status ``200``: if up and running as an asynchronous standby.
+
             * ``/health``:
+
                 * HTTP status ``200``: if up and running.
 
         .. note::
@@ -333,16 +370,16 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_OPTIONS(self) -> None:
         """Handle an ``OPTIONS`` request.
 
-        Write a simple HTTP response that represents the current PostgreSQL status. Send only `200 OK` or
+        Write a simple HTTP response that represents the current PostgreSQL status. Send only ``200 OK`` or
-        `503 Service Unavailable` as a response and nothing more, particularly no headers.
+        ``503 Service Unavailable`` as a response and nothing more, particularly no headers.
         """
         self.do_GET(write_status_code_only=True)
 
     def do_HEAD(self) -> None:
         """Handle a ``HEAD`` request.
 
-        Write a simple HTTP response that represents the current PostgreSQL status. Send only `200 OK` or
+        Write a simple HTTP response that represents the current PostgreSQL status. Send only ``200 OK`` or
-        `503 Service Unavailable` as a response and nothing more, particularly no headers.
+        ``503 Service Unavailable`` as a response and nothing more, particularly no headers.
         """
         self.do_GET(write_status_code_only=True)
 
@@ -350,11 +387,17 @@ class RestApiHandler(BaseHTTPRequestHandler):
         """Handle a ``GET`` request to ``/liveness`` path.
 
         Write a simple HTTP response with HTTP status:
+
             * ``200``:
+
                 * If the cluster is in maintenance mode; or
                 * If Patroni heartbeat loop is properly running;
-            * ``503`` if Patroni heartbeat loop last run was more than ``ttl`` setting ago on the primary (or twice the
+
+            * ``503``:
+
+                * if Patroni heartbeat loop last run was more than ``ttl`` setting ago on the primary (or twice the
                     value of ``ttl`` on a replica).
+
         """
         patroni: Patroni = self.server.patroni
         is_primary = patroni.postgresql.role in ('master', 'primary') and patroni.postgresql.is_running()
@@ -371,10 +414,14 @@ class RestApiHandler(BaseHTTPRequestHandler):
         """Handle a ``GET`` request to ``/readiness`` path.
 
         Write a simple HTTP response which HTTP status can be:
+
             * ``200``:
+
                 * If this Patroni node holds the DCS leader lock; or
                 * If this PostgreSQL instance is up and running;
+
             * ``503``: if none of the previous conditions apply.
+
         """
         patroni = self.server.patroni
         if patroni.ha.is_leader():
@@ -397,8 +444,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_GET_cluster(self) -> None:
         """Handle a ``GET`` request to ``/cluster`` path.
 
-        Write an HTTP response with JSON content based on the output of :func:`cluster_as_json`, with HTTP status
+        Write an HTTP response with JSON content based on the output of :func:`~patroni.utils.cluster_as_json`, with
-        ``200`` and the JSON representation of the cluster topology.
+        HTTP status ``200`` and the JSON representation of the cluster topology.
         """
         cluster = self.server.patroni.dcs.get_cluster(True)
         global_config = self.server.patroni.config.get_global_config(cluster)
@@ -412,11 +459,13 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         The response contains a :class:`list` of failover/switchover events. Each item is a :class:`list` with the
         following items:
+
             * Timeline when the event occurred (class:`int`);
             * LSN at which the event occurred (class:`int`);
             * The reason for the event (class:`str`);
             * Timestamp when the new timeline was created (class:`str`);
             * Name of the involved Patroni node (class:`str`).
+
         """
         cluster = self.server.patroni.dcs.cluster or self.server.patroni.dcs.get_cluster()
         self._write_json_response(200, cluster.history and cluster.history.lines or [])
@@ -448,7 +497,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
             * ``patroni_postmaster_start_time``: epoch timestamp since Postmaster was started;
             * ``patroni_master``: ``1`` if this node holds the leader lock, else ``0``;
             * ``patroni_primary``: same as ``patroni_master``;
-        * ``patroni_xlog_location``: ``pg_wal_lsn_diff(pg_current_wal_lsn(), '0/0')`` if leader, else ``0``;
+            * ``patroni_xlog_location``: ``pg_wal_lsn_diff(pg_current_wal_flush_lsn(), '0/0')`` if leader, else ``0``;
             * ``patroni_standby_leader``: ``1`` if standby leader node, else ``0``;
             * ``patroni_replica``: ``1`` if a replica, else ``0``;
             * ``patroni_sync_standby``: ``1`` if a sync replica, else ``0``;
@@ -456,7 +505,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
             * ``patroni_xlog_replayed_location``: ``pg_wal_lsn_diff(pg_last_wal_replay_lsn(), '0/0)``;
             * ``patroni_xlog_replayed_timestamp``: ``pg_last_xact_replay_timestamp``;
             * ``patroni_xlog_paused``: ``pg_is_wal_replay_paused()``;
-        * ``patroni_postgres_server_version``: Postgres version without periods, e.g. ``150002`` for Postgres ``15.2``;
+            * ``patroni_postgres_server_version``: Postgres version without periods, e.g. ``150002`` for Postgres
+              ``15.2``;
             * ``patroni_cluster_unlocked``: ``1`` if no one holds the leader lock, else ``0``;
             * ``patroni_failsafe_mode_is_active``: ``1`` if ``failsafe_mode`` is currently active, else ``0``;
             * ``patroni_postgres_timeline``: PostgreSQL timeline based on current WAL file name;
@@ -667,7 +717,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_POST_reload(self) -> None:
         """Handle a ``POST`` request to ``/reload`` path.
 
-        Schedules a reload to Patroni and writes a response with HTTP status `202`.
+        Schedules a reload to Patroni and writes a response with HTTP status ``202``.
         """
         self.server.patroni.sighup_handler()
         self.write_response(202, 'reload scheduled')
@@ -728,13 +778,17 @@ class RestApiHandler(BaseHTTPRequestHandler):
         :param schedule: a string representing a timestamp, e.g. ``2023-04-14T20:27:00+00:00``.
         :param action: the action to be scheduled (``restart``, ``switchover``, or ``failover``).
 
-        :returns: a tuple composed of 3 items
+        :returns: a tuple composed of 3 items:
+
             * Suggested HTTP status code for a response:
+
                 * ``None``: if no issue was faced while parsing, leaving it up to the caller to decide the status; or
                 * ``400``: if no timezone information could be found in *schedule*; or
                 * ``422``: if *schedule* is invalid -- in the past or not parsable.
+
             * An error message, if any error is faced, otherwise ``None``;
             * Parsed *schedule*, if able to parse, otherwise ``None``.
+
         """
         error = None
         scheduled_at = None
@@ -761,25 +815,31 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Used to restart postgres (or schedule a restart), mainly by ``patronictl restart``.
 
         The request body should be a JSON dictionary, and it can contain the following keys:
+
             * ``schedule``: timestamp at which the restart should occur;
             * ``role``: restart only nodes which role is ``role``. Can be either:
+
                 * ``primary`` (or ``master``); or
                 * ``replica``.
+
             * ``postgres_version``: restart only nodes which PostgreSQL version is less than ``postgres_version``, e.g.
                 ``15.2``;
             * ``timeout``: if restart takes longer than ``timeout`` return an error and fail over to a replica;
             * ``restart_pending``: if we should restart only when have ``pending restart`` flag;
 
         Response HTTP status codes:
+
             * ``200``: if successfully performed an immediate restart; or
             * ``202``: if successfully scheduled a restart for later; or
             * ``500``: if the cluster is in maintenance mode; or
             * ``400``: if
+
                 * ``role`` value is invalid; or
                 * ``postgres_version`` value is invalid; or
                 * ``timeout`` is not a number, or lesser than ``0``; or
                 * request contains an unknown key; or
                 * exception is faced while performing an immediate restart.
+
             * ``409``: if another restart was already previously scheduled; or
             * ``503``: if any issue was found while performing an immediate restart; or
             * HTTP status returned by :func:`parse_schedule`, if any error was observed while parsing the schedule.
@@ -857,6 +917,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Used to remove a scheduled restart of PostgreSQL.
 
         Response HTTP status codes:
+
             * ``200``: if a scheduled restart was removed; or
             * ``404``: if no scheduled restart could be found.
         """
@@ -875,6 +936,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Used to remove a scheduled switchover in the cluster.
 
         It writes a response, and the HTTP status code can be:
+
             * ``200``: if a scheduled switchover was removed; or
             * ``404``: if no scheduled switchover could be found; or
             * ``409``: if not able to update the switchover info in the DCS.
@@ -896,11 +958,13 @@ class RestApiHandler(BaseHTTPRequestHandler):
         """Handle a ``POST`` request to ``/reinitialize`` path.
 
         The request body may contain a JSON dictionary with the following key:
+
             * ``force``: ``True`` if we want to cancel an already running task in order to reinit a replica.
 
         Response HTTP status codes:
+
             * ``200``: if the reinit operation has started; or
-            * ``503``: if any error is returned by :func:`Ha.reinitialize`.
+            * ``503``: if any error is returned by :func:`~patroni.ha.Ha.reinitialize`.
         """
         request = self._read_json_content(body_is_optional=True)
 
@@ -924,11 +988,15 @@ class RestApiHandler(BaseHTTPRequestHandler):
         :param candidate: name of the Patroni node to be promoted.
         :param action: the action that is ongoing (``switchover`` or ``failover``).
 
-        :returns: a tuple composed of 2 items
+        :returns: a tuple composed of 2 items:
+
             * Response HTTP status codes:
+
                 * ``200``: if the operation succeeded; or
                 * ``503``: if the operation failed or timed out.
+
             * A status message about the operation.
+
         """
         timeout = max(10, self.server.patroni.dcs.loop_wait)
         for _ in range(0, timeout * 2):
@@ -987,12 +1055,14 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Handles manual failovers/switchovers, mainly from ``patronictl``.
 
         The request body should be a JSON dictionary, and it can contain the following keys:
+
             * ``leader``: name of the current leader in the cluster;
             * ``candidate``: name of the Patroni node to be promoted;
             * ``scheduled_at``: a string representing the timestamp when to execute the switchover/failover, e.g.
                 ``2023-04-14T20:27:00+00:00``.
 
         Response HTTP status codes:
+
             * ``202``: if operation has been scheduled;
             * ``412``: if operation is not possible;
             * ``503``: if unable to register the operation to the DCS;
@@ -1069,8 +1139,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_POST_citus(self) -> None:
         """Handle a ``POST`` request to ``/citus`` path.
 
-        Call :func:`CitusHandler.handle_event` to handle the request, then write a response with HTTP status code
+        Call :func:`~patroni.postgresql.CitusHandler.handle_event` to handle the request, then write a response with
-        ``200``.
+        HTTP status code ``200``.
 
         .. note::
             If unable to parse the request body, then the request is silently discarded.
@@ -1086,11 +1156,13 @@ class RestApiHandler(BaseHTTPRequestHandler):
         self.write_response(200, 'OK')
 
     def parse_request(self) -> bool:
-        """Override :func:`parse_request` method to enrich basic functionality of :class:`BaseHTTPRequestHandler`.
+        """Override :func:`parse_request` to enrich basic functionality of :class:`~http.server.BaseHTTPRequestHandler`.
 
         Original class can only invoke :func:`do_GET`, :func:`do_POST`, :func:`do_PUT`, etc method implementations if
         they are defined.
+
         But we would like to have at least some simple routing mechanism, i.e.:
+
             * ``GET /uri1/part2`` request should invoke :func:`do_GET_uri1()`
             * ``POST /other`` should invoke :func:`do_POST_other()`
 
@@ -1098,6 +1170,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         :returns: ``True`` for success, ``False`` for failure; on failure, any relevant error response has already been
                   sent back.
+
         """
         ret = BaseHTTPRequestHandler.parse_request(self)
         if ret:
@@ -1131,7 +1204,9 @@ class RestApiHandler(BaseHTTPRequestHandler):
         Some of the values are collected by executing a query and other are taken from the state stored in memory.
 
         :param retry: whether the query should be retried if failed or give up immediately
+
         :returns: a dict with the status of Postgres/Patroni. The keys are:
+
             * ``state``: Postgres state among ``stopping``, ``stopped``, ``stop failed``, ``crashed``, ``running``,
               ``starting``, ``start failed``, ``restarting``, ``restart failed``, ``initializing new cluster``,
               ``initdb failed``, ``running custom bootstrap script``, ``custom bootstrap failed``,
@@ -1140,27 +1215,35 @@ class RestApiHandler(BaseHTTPRequestHandler):
             * ``role``: ``replica`` or ``master`` based on ``pg_is_in_recovery()`` output;
             * ``server_version``: Postgres version without periods, e.g. ``150002`` for Postgres ``15.2``;
             * ``xlog``: dictionary. Its structure depends on ``role``:
+
                 * If ``master``:
-                    * ``location``: ``pg_current_wal_lsn()``
+
+                    * ``location``: ``pg_current_wal_flush_lsn()``
+
                 * If ``replica``:
+
                     * ``received_location``: ``pg_wal_lsn_diff(pg_last_wal_receive_lsn(), '0/0')``;
                     * ``replayed_location``: ``pg_wal_lsn_diff(pg_last_wal_replay_lsn(), '0/0)``;
                     * ``replayed_timestamp``: ``pg_last_xact_replay_timestamp``;
                     * ``paused``: ``pg_is_wal_replay_paused()``;
+
             * ``sync_standby``: ``True`` if replication mode is synchronous and this is a sync standby;
             * ``timeline``: PostgreSQL primary node timeline;
             * ``replication``: :class:`list` of :class:`dict` entries, one for each replication connection. Each entry
                 contains the following keys:
+
                 * ``application_name``: ``pg_stat_activity.application_name``;
                 * ``client_addr``: ``pg_stat_activity.client_addr``;
                 * ``state``: ``pg_stat_replication.state``;
                 * ``sync_priority``: ``pg_stat_replication.sync_priority``;
                 * ``sync_state``: ``pg_stat_replication.sync_state``;
                 * ``usename``: ``pg_stat_activity.usename``.
+
             * ``pause``: ``True`` if cluster is in maintenance mode;
             * ``cluster_unlocked``: ``True`` if cluster has no node holding the leader lock;
             * ``failsafe_mode_is_active``: ``True`` if DCS failsafe mode is currently active;
             * ``dcs_last_seen``: epoch timestamp DCS was last reached by Patroni.
+
         """
         postgresql = self.server.patroni.postgresql
         cluster = self.server.patroni.dcs.cluster
@@ -1291,8 +1374,10 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         :param params: positional arguments to be used as parameters for *sql*.
 
         :returns: a list of rows that were fetched from the database.
-        :raises psycopg.Error: if had issues while executing *sql*.
+
-        :raises PostgresConnectionException: if had issues while connecting to the database.
+        :raises:
+            :class:`psycopg.Error`: if had issues while executing *sql*.
+            :class:`~patroni.exceptions.PostgresConnectionException`: if had issues while connecting to the database.
         """
         cursor = None
         try:
@@ -1352,7 +1437,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         :param host: hostname to be checked.
         :param port: port to be checked.
 
-        :rtype: Iterator[Union[IPv4Network, IPv6Network]] of *host* + *port* resolved to IP networks.
+        :yields: *host* + *port* resolved to IP networks.
         """
         try:
             for _, _, _, _, sa in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP):
@@ -1366,8 +1451,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         .. note::
             Only yields object if ``restapi.allowlist_include_members`` setting is enabled.
 
-        :rtype: Iterator[Union[IPv4Network, IPv6Network]] of each node ``restapi.connect_address`` resolved to an IP
+        :yields: each node ``restapi.connect_address`` resolved to an IP network.
-            network.
         """
         cluster = self.patroni.dcs.cluster
         if self.__allowlist_include_members and cluster:
@@ -1387,8 +1471,10 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         """Ensure client has enough privileges to perform a given request.
 
         Write a response back to the client if any issue is observed, and the HTTP status may be:
+
             * ``401``: if ``Authorization`` header is missing or contain an invalid password;
             * ``403``: if:
+
                 * ``restapi.allowlist`` was configured, but client IP is not in the allowed list; or
                 * ``restapi.allowlist_include_members`` is enabled, but client IP is not in the members list; or
                 * a client certificate is expected by the server, but is missing in the request.
@@ -1468,17 +1554,20 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
             ``host`` can be a hostname or IP address. It is the value of ``restapi.listen`` setting.
         :param ssl_options: dictionary that may contain the following keys, depending on what has been configured in
             ``restapi` section:
+
             * ``certfile``: path to PEM certificate. If given, will start in HTTPS mode;
             * ``keyfile``: path to key of ``certfile``;
             * ``keyfile_password``: password for decrypting ``keyfile``;
             * ``cafile``: path to CA file to validate client certificates;
             * ``ciphers``: permitted cipher suites;
             * ``verify_client``: value can be one among:
+
                 * ``none``: do not check client certificates;
                 * ``optional``: check client certificate only for unsafe REST API endpoints;
                 * ``required``: check client certificate for all REST API endpoints.
 
-        :raises ValueError: if any issue is faced while parsing *listen*.
+        :raises:
+            :class:`ValueError`: if any issue is faced while parsing *listen*.
         """
         try:
             host, port = split_host_port(listen, None)
@@ -1526,7 +1615,8 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
                                client_address: Tuple[str, int]) -> None:
         """Process a request to the REST API.
 
-        Wrapper for :func:`ThreadingMixIn.process_request_thread` that additionally:
+        Wrapper for :func:`~socketserver.ThreadingMixIn.process_request_thread` that additionally:
+
             * Enable TCP keepalive
             * Perform SSL handshake (if an SSL socket).
 
@@ -1544,7 +1634,8 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
     def shutdown_request(self, request: Union[socket.socket, Tuple[bytes, socket.socket]]) -> None:
         """Shut down a request to the REST API.
 
-        Wrapper for :func:`HTTPServer.shutdown_request` that additionally:
+        Wrapper for :func:`http.server.HTTPServer.shutdown_request` that additionally:
+
             * Perform SSL shutdown handshake (if a SSL socket).
 
         :param request: socket to handle the client request.
@@ -1592,7 +1683,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         :param value: list of IPs and/or networks contained in ``restapi.allowlist`` setting. Each item can be a host,
             an IP, or a network in CIDR format.
 
-        :rtype: Iterator[Union[IPv4Network, IPv6Network]] of *host* + *port* resolved to IP networks.
+        :yields: *host* + *port* resolved to IP networks.
         """
         if isinstance(value, list):
             for v in value:
@@ -1609,7 +1700,9 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         """Reload REST API configuration.
 
         :param config: dictionary representing values under the ``restapi`` configuration section.
-        :raises ValueError: if ``listen`` key is not present in *config*.
+
+        :raises:
+            :class:`ValueError`: if ``listen`` key is not present in *config*.
         """
         if 'listen' not in config:  # changing config in runtime
             raise ValueError('Can not find "restapi.listen" config')

patroni/config.py

@@ -15,6 +15,7 @@ from .dcs import ClusterConfig, Cluster
 from .exceptions import ConfigParseError
 from .file_perm import pg_perm
 from .postgresql.config import ConfigHandler
+from .validator import IntValidator
 from .utils import deep_compare, parse_bool, parse_int, patch_config
 
 logger = logging.getLogger(__name__)
@@ -224,6 +225,7 @@ class Config(object):
         self.__effective_configuration = self._build_effective_configuration({}, self._local_configuration)
         self._data_dir = self.__effective_configuration.get('postgresql', {}).get('data_dir', "")
         self._cache_file = os.path.join(self._data_dir, self.__CACHE_FILENAME)
+        if validator:  # patronictl uses validator=None and we don't want to load anything from local cache in this case
             self._load_cache()
         self._cache_needs_saving = False
 
@@ -338,10 +340,12 @@ class Config(object):
             if name not in ConfigHandler.CMDLINE_OPTIONS:
                 pg_params[name] = value
             elif not is_local:
-                if ConfigHandler.CMDLINE_OPTIONS[name][1](value):
+                validator = ConfigHandler.CMDLINE_OPTIONS[name][1]
-                    pg_params[name] = value
+                if validator(value):
+                    int_val = parse_int(value) if isinstance(validator, IntValidator) else None
+                    pg_params[name] = int_val if isinstance(int_val, int) else value
                 else:
-                    logging.warning("postgresql parameter %s=%s failed validation, defaulting to %s",
+                    logger.warning("postgresql parameter %s=%s failed validation, defaulting to %s",
                                    name, value, ConfigHandler.CMDLINE_OPTIONS[name][0])
 
         return pg_params

patroni/ctl.py

@@ -264,7 +264,9 @@ role_choice = click.Choice(['leader', 'primary', 'standby-leader', 'replica', 's
 @click.option('-k', '--insecure', is_flag=True, help='Allow connections to SSL sites without certs')
 @click.pass_context
 def ctl(ctx: click.Context, config_file: str, dcs_url: Optional[str], insecure: bool) -> None:
-    """Entry point of ``patronictl`` utility.
+    """Command-line interface for interacting with Patroni.
+    \f
+    Entry point of ``patronictl`` utility.
 
     Load the configuration file.
 
@@ -560,9 +562,10 @@ def get_cursor(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], conn
     from . import psycopg
     conn = psycopg.connect(**params)
     cursor = conn.cursor()
-    # If we want ``any`` node we are fine to return the cursor
+    # If we want ``any`` node we are fine to return the cursor. ``None`` is similar to ``any`` at this point, as it's
+    # been dealt with through :func:`get_any_member`.
     # If we want the Patroni leader node, :func:`get_any_member` already checks that for us
-    if role in ('any', 'leader'):
+    if role in (None, 'any', 'leader'):
         return cursor
 
     # If we want something other than ``any`` or ``leader``, then we do not rely only on the DCS information about
@@ -856,9 +859,11 @@ def query_member(obj: Dict[str, Any], cluster: Cluster, group: Optional[int],
 
         if cursor is None:
             if member is not None:
-                message = 'No connection to member {0} is available'.format(member)
+                message = f'No connection to member {member} is available'
+            elif role is not None:
+                message = f'No connection to role {role} is available'
             else:
-                message = 'No connection to role={0} is available'.format(role)
+                message = 'No connection is available'
             logging.debug(message)
             return [[timestamp(0), message]], None
 
@@ -1561,9 +1566,11 @@ def output_members(obj: Dict[str, Any], cluster: Cluster, name: str,
             rows.append([member.get(n.lower().replace(' ', '_'), '') for n in columns])
 
     title = 'Citus cluster' if is_citus_cluster else 'Cluster'
-    group_title = '' if group is None else 'group: {0}, '.format(group)
+    title_details = f' ({initialize})'
-    title_details = group_title and ' ({0}{1})'.format(group_title, initialize)
+    if is_citus_cluster:
-    title = ' {0}: {1}{2} '.format(title, name, title_details)
+        title_details = '' if group is None else f' (group: {group}, {initialize})'
+
+    title = f' {title}: {name}{title_details} '
     print_output(columns, rows, {'Group': 'r', 'Lag in MB': 'r', 'TL': 'r'}, fmt, title)
 
     if fmt not in ('pretty', 'topology'):  # Omit service info when using machine-readable formats

patroni/dcs/etcd3.py

@@ -205,7 +205,7 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
 
     def __init__(self, config: Dict[str, Any], dns_resolver: DnsCachingResolver, cache_ttl: int = 300) -> None:
         self._token = None
-        self._cluster_version: Tuple[int] = tuple()
+        self._cluster_version: Tuple[int, ...] = tuple()
         super(Etcd3Client, self).__init__({**config, 'version_prefix': '/v3beta'}, dns_resolver, cache_ttl)
 
         try:

patroni/dcs/kubernetes.py

@@ -756,7 +756,7 @@ class Kubernetes(AbstractDCS):
         self._role_label = config.get('role_label', 'role')
         self._leader_label_value = config.get('leader_label_value', 'master')
         self._follower_label_value = config.get('follower_label_value', 'replica')
-        self._standby_leader_label_value = config.get('standby_leader_label_value', 'standby-leader')
+        self._standby_leader_label_value = config.get('standby_leader_label_value', 'master')
         self._tmp_role_label = config.get('tmp_role_label')
         self._ca_certs = os.environ.get('PATRONI_KUBERNETES_CACERT', config.get('cacert')) or SERVICE_CERT_FILENAME
         super(Kubernetes, self).__init__({**config, 'namespace': ''})
@@ -836,7 +836,7 @@ class Kubernetes(AbstractDCS):
         self._api.configure_timeouts(self.loop_wait, self._retry.deadline, self.ttl)
 
         # retriable_http_codes supposed to be either int, list of integers or comma-separated string with integers.
-        retriable_http_codes = config.get('retriable_http_codes', [])
+        retriable_http_codes: Union[str, List[Union[str, int]]] = config.get('retriable_http_codes', [])
         if not isinstance(retriable_http_codes, list):
             retriable_http_codes = [c.strip() for c in str(retriable_http_codes).split(',')]
 
@@ -1140,6 +1140,13 @@ class Kubernetes(AbstractDCS):
         """Unused"""
         raise NotImplementedError  # pragma: no cover
 
+    def write_leader_optime(self, last_lsn: int) -> None:
+        """Write value for WAL LSN to ``optime`` annotation of the leader object.
+
+        :param last_lsn: absolute WAL LSN in bytes.
+        """
+        self.patch_or_create(self.leader_path, {self._OPTIME: str(last_lsn)}, patch=True, retry=False)
+
     def _update_leader_with_retry(self, annotations: Dict[str, Any],
                                   resource_version: Optional[str], ips: List[str]) -> bool:
         retry = self._retry.copy()
@@ -1269,13 +1276,10 @@ class Kubernetes(AbstractDCS):
     def touch_member(self, data: Dict[str, Any]) -> bool:
         cluster = self.cluster
         if cluster and cluster.leader and cluster.leader.name == self._name:
-            role = self._leader_label_value
+            role = self._standby_leader_label_value if data['role'] == 'standby_leader' else self._leader_label_value
             tmp_role = 'master'
         elif data['state'] == 'running' and data['role'] not in ('master', 'primary'):
-            role = {
+            role = {'replica': self._follower_label_value}.get(data['role'], data['role'])
-                'replica': self._follower_label_value,
-                'standby-leader': self._standby_leader_label_value,
-            }.get(data['role'], data['role'])
             tmp_role = data['role']
         else:
             role = None

patroni/dcs/zookeeper.py

@@ -89,7 +89,7 @@ class ZooKeeper(AbstractDCS):
     def __init__(self, config: Dict[str, Any]) -> None:
         super(ZooKeeper, self).__init__(config)
 
-        hosts = config.get('hosts', [])
+        hosts: Union[str, List[str]] = config.get('hosts', [])
         if isinstance(hosts, list):
             hosts = ','.join(hosts)
 
@@ -393,14 +393,21 @@ class ZooKeeper(AbstractDCS):
         cluster = self.cluster
         member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
         member_data = self.__last_member_data or member and member.data
-        #  We want to notify leader if some important fields in the member key changed by removing ZNode
+        if member and member_data:
-        if member and (self._client.client_id is not None and member.session != self._client.client_id[0]
+            is_leader = data.get('role') in ('master', 'primary', 'standby_leader')
-                       or not (member_data and deep_compare(member_data.get('tags', {}), data.get('tags', {}))
+            checkpoint_after_promote_changed = member_data.get('checkpoint_after_promote') \
-                               and (member_data.get('state') == data.get('state')
+                != data.get('checkpoint_after_promote')
-                                    or 'running' not in (member_data.get('state'), data.get('state')))
+            state_running_changed = member_data.get('state') != data.get('state') \
-                               and member_data.get('version') == data.get('version')
+                and 'running' in (member_data.get('state'), data.get('state'))
-                               and member_data.get('checkpoint_after_promote')
+            tags_changed = not deep_compare(member_data.get('tags', {}), data.get('tags', {}))
-                               == data.get('checkpoint_after_promote'))):
+
+            # We want delete the member ZNode if:
+            # - our session doesn't match with session id on our member key; or
+            # - we want to notify leader if some important fields in the member key changed; or
+            # - if we are the leader and want to notify replicas about checkpoint_after_promote;
+            if self._client.client_id is not None and member.session != self._client.client_id[0] \
+                    or is_leader and checkpoint_after_promote_changed \
+                    or not is_leader and (state_running_changed or tags_changed):
                 try:
                     self._client.delete_async(self.member_path).get(timeout=1)
                 except NoNodeError:

patroni/ha.py

@@ -83,11 +83,7 @@ class Failsafe(object):
     def __init__(self, dcs: AbstractDCS) -> None:
         self._lock = RLock()
         self._dcs = dcs
-        self._last_update = 0
+        self._reset_state()
-        self._name = None
-        self._conn_url = None
-        self._api_url = None
-        self._slots = None
 
     def update(self, data: Dict[str, Any]) -> None:
         with self._lock:
@@ -97,6 +93,13 @@ class Failsafe(object):
             self._api_url = data['api_url']
             self._slots = data.get('slots')
 
+    def _reset_state(self) -> None:
+        self._last_update = 0
+        self._name = None
+        self._conn_url = None
+        self._api_url = None
+        self._slots = None
+
     @property
     def leader(self) -> Optional[Leader]:
         with self._lock:
@@ -130,6 +133,8 @@ class Failsafe(object):
     def set_is_active(self, value: float) -> None:
         with self._lock:
             self._last_update = value
+            if not value:
+                self._reset_state()
 
 
 class Ha(object):
@@ -442,15 +447,22 @@ class Ha(object):
         """Handle the case when postgres isn't running.
 
         Depending on the state of Patroni, DCS cluster view, and pg_controldata the following could happen:
+
           - if ``primary_start_timeout`` is 0 and this node owns the leader lock, the lock
             will be voluntarily released if there are healthy replicas to take it over.
+
           - if postgres was running as a ``primary`` and this node owns the leader lock, postgres is started as primary.
+
           - crash recover in a single-user mode is executed in the following cases:
+
             - postgres was running as ``primary`` wasn't ``shut down`` cleanly and there is no leader in DCS
+
             - postgres was running as ``replica`` wasn't ``shut down in recovery`` (cleanly)
               and we need to run ``pg_rewind`` to join back to the cluster.
+
           - ``pg_rewind`` is executed if it is necessary, or optinally, the data directory could
              be removed if it is allowed by configuration.
+
           - after ``crash recovery`` and/or ``pg_rewind`` are executed, postgres is started in recovery.
 
         :returns: action message, describing what was performed.
@@ -629,11 +641,28 @@ class Ha(object):
         promoting standbys that were guaranteed to be replicating synchronously.
         """
         if self.is_synchronous_mode():
-            current = CaseInsensitiveSet(self.cluster.sync.members)
+            sync = self.cluster.sync
+            if sync.is_empty:
+                # corner case: we need to explicitly enable synchronous mode by updating the
+                # ``/sync`` key with the current leader name and empty members. In opposite case
+                # it will never be automatically enabled if there are not eligible candidates.
+                sync = self.dcs.write_sync_state(self.state_handler.name, None, version=sync.version)
+                if not sync:
+                    return logger.warning("Updating sync state failed")
+                logger.info("Enabled synchronous replication")
+
+            current = CaseInsensitiveSet(sync.members)
             picked, allow_promote = self.state_handler.sync_handler.current_state(self.cluster)
 
+            if picked == current and current != allow_promote:
+                logger.warning('Inconsistent state between synchronous_standby_names = %s and /sync = %s key '
+                               'detected, updating synchronous replication key...', list(allow_promote), list(current))
+                sync = self.dcs.write_sync_state(self.state_handler.name, allow_promote, version=sync.version)
+                if not sync:
+                    return logger.warning("Updating sync state failed")
+                current = CaseInsensitiveSet(sync.members)
+
             if picked != current:
-                sync = self.cluster.sync
                 # update synchronous standby list in dcs temporarily to point to common nodes in current and picked
                 sync_common = current & allow_promote
                 if sync_common != current:
@@ -714,13 +743,13 @@ class Ha(object):
             if cluster_history:
                 self.dcs.set_history_value('[]')
         elif not cluster_history or cluster_history[-1][0] != primary_timeline - 1 or len(cluster_history[-1]) != 5:
-            cluster_history = {line[0]: line for line in cluster_history}
+            cluster_history_dict: Dict[int, List[Any]] = {line[0]: list(line) for line in cluster_history}
             history: List[List[Any]] = list(map(list, self.state_handler.get_history(primary_timeline)))
             if self.cluster.config:
                 history = history[-self.cluster.config.max_timelines_history:]
             for line in history:
                 # enrich current history with promotion timestamps stored in DCS
-                cluster_history_line = list(cluster_history.get(line[0], []))
+                cluster_history_line = cluster_history_dict.get(line[0], [])
                 if len(line) == 3 and len(cluster_history_line) >= 4 and cluster_history_line[1] == line[1]:
                     line.append(cluster_history_line[3])
                     if len(cluster_history_line) == 5:
@@ -777,6 +806,9 @@ class Ha(object):
                 self.state_handler.sync_handler.set_synchronous_standby_names(
                     CaseInsensitiveSet('*') if self.global_config.is_synchronous_mode_strict else CaseInsensitiveSet())
             if self.state_handler.role not in ('master', 'promoted', 'primary'):
+                # reset failsafe state when promote
+                self._failsafe.set_is_active(0)
+
                 def before_promote():
                     self.notify_citus_coordinator('before_promote')
 
@@ -854,6 +886,7 @@ class Ha(object):
         """Returns if instance with an wal should consider itself unhealthy to be promoted due to replication lag.
 
         :param wal_position: Current wal position.
+
         :returns True when node is lagging
         """
         lag = (self.cluster.last_lsn or 0) - wal_position
@@ -990,6 +1023,7 @@ class Ha(object):
         """Performs a series of checks to determine that the current node is the best candidate.
 
         In case if manual failover/switchover is requested it calls :func:`manual_failover_process_no_leader` method.
+
         :returns: `True` if the current node is among the best candidates to become the new leader.
         """
         if time.time() - self._released_leader_key_timestamp < self.dcs.ttl:
@@ -1074,13 +1108,15 @@ class Ha(object):
     def demote(self, mode: str) -> Optional[bool]:
         """Demote PostgreSQL running as primary.
 
-        :param mode: One of offline, graceful or immediate.
+        :param mode: One of offline, graceful, immediate or immediate-nolock.
-            offline is used when connection to DCS is not available.
+                     ``offline`` is used when connection to DCS is not available.
-            graceful is used when failing over to another node due to user request. May only be called running async.
+                     ``graceful`` is used when failing over to another node due to user request. May only be called
-            immediate is used when we determine that we are not suitable for primary and want to failover quickly
+                     running async.
-                without regard for data durability. May only be called synchronously.
+                     ``immediate`` is used when we determine that we are not suitable for primary and want to failover
-            immediate-nolock is used when find out that we have lost the lock to be primary. Need to bring down
+                     quickly without regard for data durability. May only be called synchronously.
-                PostgreSQL as quickly as possible without regard for data durability. May only be called synchronously.
+                     ``immediate-nolock`` is used when find out that we have lost the lock to be primary. Need to bring
+                     down PostgreSQL as quickly as possible without regard for data durability. May only be called
+                     synchronously.
         """
         mode_control = {
             'offline':          dict(stop='fast',      checkpoint=False, release=False, offline=True,  async_req=False),  # noqa: E241,E501
@@ -1472,9 +1508,7 @@ class Ha(object):
         self._async_executor.run_async(self._do_reinitialize, args=(cluster, ))
 
     def handle_long_action_in_progress(self) -> str:
-        """
+        """Figure out what to do with the task AsyncExecutor is performing."""
-        Figure out what to do with the task AsyncExecutor is performing.
-        """
         if self.has_lock() and self.update_lock():
             if self._async_executor.scheduled_action == 'doing crash recovery in a single user mode':
                 time_left = self.global_config.primary_start_timeout - (time.time() - self._crash_recovery_started)
@@ -1569,8 +1603,7 @@ class Ha(object):
         return 'initialized a new cluster'
 
     def handle_starting_instance(self) -> Optional[str]:
-        """Starting up PostgreSQL may take a long time. In case we are the leader we may want to
+        """Starting up PostgreSQL may take a long time. In case we are the leader we may want to fail over to."""
-        fail over to."""
 
         # Check if we are in startup, when paused defer to main loop for manual failovers.
         if not self.state_handler.check_for_startup() or self.is_paused():
@@ -1610,7 +1643,8 @@ class Ha(object):
     def set_start_timeout(self, value: Optional[int]) -> None:
         """Sets timeout for starting as primary before eligible for failover.
 
-        Must be called when async_executor is busy or in the main thread."""
+        Must be called when async_executor is busy or in the main thread.
+        """
         self._start_timeout = value
 
     def _run_cycle(self) -> str:
@@ -1815,7 +1849,9 @@ class Ha(object):
         """Handles replication slots.
 
         :param dcs_failed: bool, indicates that communication with DCS failed (get_cluster() or update_leader())
-        :returns: list[str], replication slots names that should be copied from the primary"""
+
+        :returns: list[str], replication slots names that should be copied from the primary
+        """
 
         slots: List[str] = []
 
@@ -1906,15 +1942,16 @@ class Ha(object):
         return self.dcs.watch(leader_version, timeout)
 
     def wakeup(self) -> None:
-        """Call of this method will trigger the next run of HA loop if there is
+        """Trigger the next run of HA loop if there is no "active" leader watch request in progress.
-        no "active" leader watch request in progress.
+
         This usually happens on the leader or if the node is running async action"""
         self.dcs.event.set()
 
     def get_remote_member(self, member: Union[Leader, Member, None] = None) -> RemoteMember:
-        """ In case of standby cluster this will tel us from which remote
+        """Get remote member node to stream from.
-            member to stream. Config can be both patroni config or
+
-            cluster.config.data
+        In case of standby cluster this will tell us from which remote member to stream. Config can be both patroni
+        config or cluster.config.data.
         """
         data: Dict[str, Any] = {}
         cluster_params = self.global_config.get_standby_cluster_config()

patroni/log.py

@@ -21,17 +21,17 @@ _LOGGER = logging.getLogger(__name__)
 def debug_exception(self: logging.Logger, msg: object, *args: Any, **kwargs: Any) -> None:
     """Add full stack trace info to debug log messages and partial to others.
 
-    Handle :func:`exception` calls for *self*.
+    Handle :func:`~self.exception` calls for *self*.
 
     .. note::
         * If *self* log level is set to ``DEBUG``, then issue a ``DEBUG`` message with the complete stack trace;
         * If *self* log level is ``INFO`` or higher, then issue an ``ERROR`` message with only the last line of
             the stack trace.
 
-    :param self: logger for which :func:`exception` will be processed.
+    :param self: logger for which :func:`~self.exception` will be processed.
     :param msg: the message related to the exception to be logged.
-    :param args: positional arguments to be passed to :func:`self.debug` or :func:`loger_obj.error`.
+    :param args: positional arguments to be passed to :func:`~self.debug` or :func:`~self.error`.
-    :param kwargs: keyword arguments to be passed to :func:`self.debug` or :func:`loger_obj.error`.
+    :param kwargs: keyword arguments to be passed to :func:`~self.debug` or :func:`~self.error`.
     """
     kwargs.pop("exc_info", False)
     if self.isEnabledFor(logging.DEBUG):
@@ -44,16 +44,16 @@ def debug_exception(self: logging.Logger, msg: object, *args: Any, **kwargs: Any
 def error_exception(self: logging.Logger, msg: object, *args: Any, **kwargs: Any) -> None:
     """Add full stack trace info to error messages.
 
-    Handle :func:`exception` calls for *self*.
+    Handle :func:`~self.exception` calls for *self*.
 
     .. note::
         * By default issue an ``ERROR`` message with the complete stack trace. If you do not want to show the complete
           stack trace, call with ``exc_info=False``.
 
-    :param self: logger for which :func:`exception` will be processed.
+    :param self: logger for which :func:`~self.exception` will be processed.
     :param msg: the message related to the exception to be logged.
-    :param args: positional arguments to be passed to :func:`loger_obj.error`.
+    :param args: positional arguments to be passed to :func:`~self.error`.
-    :param kwargs: keyword arguments to be passed to :func:`loger_obj.error`.
+    :param kwargs: keyword arguments to be passed to :func:`~self.error`.
     """
     exc_info = kwargs.pop("exc_info", True)
     self.error(msg, *args, exc_info=exc_info, **kwargs)
@@ -140,7 +140,7 @@ class ProxyHandler(logging.Handler):
     def emit(self, record: logging.LogRecord) -> None:
         """Emit each log record that is handled.
 
-        Will push the log record down to :func:`handle` method of the currently configured log handler.
+        Will push the log record down to :func:`~logging.Handler.handle` method of the currently configured log handler.
 
         :param record: the record that was emitted.
         """
@@ -203,7 +203,7 @@ class PatroniLogger(Thread):
         self._root_logger.addHandler(self._proxy_handler)
 
     def update_loggers(self) -> None:
-        """Configure loggers' log level as defined in ``log.loggers` section of Patroni configuration.
+        """Configure loggers' log level as defined in ``log.loggers`` section of Patroni configuration.
 
         .. note::
             It creates logger objects that are not defined yet in the log manager.
@@ -281,7 +281,8 @@ class PatroniLogger(Thread):
 
         .. note::
             It is used to remove different handlers that were configured previous to a reload in the configuration,
-            e.g. if we are switching from :class:`RotatingFileHandler` to class:`StreamHandler` and vice-versa.
+            e.g. if we are switching from :class:`~logging.handlers.RotatingFileHandler` to
+            class:`~logging.StreamHandler` and vice-versa.
         """
         while True:
             with self.log_handler_lock:

patroni/postgresql/__init__.py

@@ -118,17 +118,28 @@ class Postgresql(object):
         # Last known running process
         self._postmaster_proc = None
 
-        if self.is_running():  # we are "joining" already running postgres
+        if self.is_running():
-            self.set_state('running')
+            # If we found postmaster process we need to figure out whether postgres is accepting connections
+            self.set_state('starting')
+            self.check_startup_state_changed()
+
+        if self.state == 'running':  # we are "joining" already running postgres
+            # we know that PostgreSQL is accepting connections and can read some GUC's from pg_settings
+            self.config.load_current_server_parameters()
+
             self.set_role('master' if self.is_leader() else 'replica')
-            # postpone writing postgresql.conf for 12+ because recovery parameters are not yet known
+
-            if self.major_version < 120000 or self.is_leader():
-                self.config.write_postgresql_conf()
             hba_saved = self.config.replace_pg_hba()
             ident_saved = self.config.replace_pg_ident()
-            if hba_saved or ident_saved:
+
+            if self.major_version < 120000 or self.role in ('master', 'primary'):
+                # If PostgreSQL is running as a primary or we run PostgreSQL that is older than 12 we can
+                # call reload_config() once again (the first call happened in the ConfigHandler constructor),
+                # so that it can figure out if config files should be updated and pg_ctl reload executed.
+                self.config.reload_config(config, sighup=bool(hba_saved or ident_saved))
+            elif hba_saved or ident_saved:
                 self.reload()
-        elif self.role in ('master', 'primary'):
+        elif not self.is_running() and self.role in ('master', 'primary'):
             self.set_role('demoted')
 
     @property
@@ -173,6 +184,7 @@ class Postgresql(object):
         """Returns the monitoring query with a fixed number of fields.
 
         The query text is constructed based on current state in DCS and PostgreSQL version:
+
         1. function names depend on version. wal/lsn for v10+ and xlog/location for pre v10.
         2. for primary we query timeline_id (extracted from pg_walfile_name()) and pg_current_wal_lsn()
         3. for replicas we query pg_last_wal_receive_lsn(), pg_last_wal_replay_lsn(), and  pg_is_wal_replay_paused()
@@ -182,7 +194,8 @@ class Postgresql(object):
         7. if sync replication is enabled we query pg_stat_replication and aggregate the result.
            In addition to that we get current values of synchronous_commit and synchronous_standby_names GUCs.
 
-        If some conditions are not satisfied we simply put static values instead. E.g., NULL, 0, '', and so on."""
+        If some conditions are not satisfied we simply put static values instead. E.g., NULL, 0, '', and so on.
+        """
 
         extra = ", " + (("pg_catalog.current_setting('synchronous_commit'), "
                          "pg_catalog.current_setting('synchronous_standby_names'), "
@@ -408,7 +421,18 @@ class Postgresql(object):
         :param global_config: last known :class:`GlobalConfig` object
         """
         self._cluster_info_state = {}
-        if cluster and cluster.config and cluster.config.modify_version:
+
+        if global_config:
+            self._global_config = global_config
+
+        if not self._global_config:
+            return
+
+        if self._global_config.is_standby_cluster:
+            # Standby cluster can't have logical replication slots, and we don't need to enforce hot_standby_feedback
+            self._has_permanent_logical_slots = False
+            self.set_enforce_hot_standby_feedback(False)
+        elif cluster and cluster.config and cluster.config.modify_version:
             self._has_permanent_logical_slots =\
                 cluster.has_permanent_logical_slots(self.name, nofailover, self.major_version)
 
@@ -418,9 +442,6 @@ class Postgresql(object):
                 self._has_permanent_logical_slots
                 or cluster.should_enforce_hot_standby_feedback(self.name, nofailover, self.major_version))
 
-        if global_config:
-            self._global_config = global_config
-
     def _cluster_info_state_get(self, name: str) -> Optional[Any]:
         if not self._cluster_info_state:
             try:
@@ -542,7 +563,7 @@ class Postgresql(object):
                              r'lsn: ([0-9A-Fa-f]+/[0-9A-Fa-f]+), prev ([0-9A-Fa-f]+/[0-9A-Fa-f]+), '
                              r'.*?desc: (.+)', out.decode('utf-8'))
             if match:
-                return match.groups()
+                return match.group(1), match.group(2), match.group(3), match.group(4)
         return None, None, None, None
 
     def latest_checkpoint_location(self) -> Optional[int]:
@@ -998,7 +1019,7 @@ class Postgresql(object):
             return None, None
 
     @contextmanager
-    def get_replication_connection_cursor(self, host: Optional[str] = None, port: int = 5432,
+    def get_replication_connection_cursor(self, host: Optional[str] = None, port: Union[int, str] = 5432,
                                           **kwargs: Any) -> Iterator[Union['cursor', 'Cursor[Any]']]:
         conn_kwargs = self.config.replication.copy()
         conn_kwargs.update(host=host, port=int(port) if port else None, user=conn_kwargs.pop('username'),

patroni/postgresql/citus.py

@@ -174,11 +174,13 @@ class CitusHandler(Thread):
         """Returns the tuple(i, task), where `i` - is the task index in the self._tasks list
 
         Tasks are picked by following priorities:
+
         1. If there is already a transaction in progress, pick a task
            that that will change already affected worker primary.
         2. If the coordinator address should be changed - pick a task
            with group=0 (coordinators are always in group 0).
-        3. Pick a task that is the oldest (first from the self._tasks)"""
+        3. Pick a task that is the oldest (first from the self._tasks)
+        """
 
         with self._condition:
             if self._in_flight:
@@ -405,7 +407,7 @@ class CitusHandler(Thread):
         parameters['shared_preload_libraries'] = ','.join(['citus'] + shared_preload_libraries)
 
         # if not explicitly set Citus overrides max_prepared_transactions to max_connections*2
-        if parameters.get('max_prepared_transactions') == 0:
+        if parameters['max_prepared_transactions'] == 0:
             parameters['max_prepared_transactions'] = parameters['max_connections'] * 2
 
         # Resharding in Citus implemented using logical replication

patroni/postgresql/config.py

@@ -14,7 +14,7 @@ from typing import Any, Collection, Dict, Iterator, List, Optional, Union, Tuple
 from .validator import recovery_parameters, transform_postgresql_parameter_value, transform_recovery_parameter_value
 from ..collections import CaseInsensitiveDict, CaseInsensitiveSet
 from ..dcs import Leader, Member, RemoteMember, slot_name_from_member_name
-from ..exceptions import PatroniFatalException
+from ..exceptions import PatroniFatalException, PostgresConnectionException
 from ..file_perm import pg_perm
 from ..utils import compare_values, parse_bool, parse_int, split_host_port, uri, validate_directory, is_subpath
 from ..validator import IntValidator, EnumValidator
@@ -326,14 +326,22 @@ class ConfigHandler(object):
                                         .format(self._pgpass))
         self._passfile = None
         self._passfile_mtime = None
-        self._synchronous_standby_names = None
         self._postmaster_ctime = None
         self._current_recovery_params: Optional[CaseInsensitiveDict] = None
         self._config = {}
         self._recovery_params = CaseInsensitiveDict()
-        self._server_parameters: CaseInsensitiveDict
+        self._server_parameters: CaseInsensitiveDict = CaseInsensitiveDict()
         self.reload_config(config)
 
+    def load_current_server_parameters(self) -> None:
+        """Read GUC's values from ``pg_settings`` when Patroni is joining the the postgres that is already running."""
+        exclude = [name.lower() for name, value in self.CMDLINE_OPTIONS.items() if value[1] == _false_validator] \
+            + [name.lower() for name in self._RECOVERY_PARAMETERS]
+        self._server_parameters = CaseInsensitiveDict({r[0]: r[1] for r in self._postgresql.query(
+            "SELECT name, pg_catalog.current_setting(name) FROM pg_catalog.pg_settings"
+            " WHERE (source IN ('command line', 'environment variable') OR sourcefile = %s)"
+            " AND pg_catalog.lower(name) != ALL(%s)", self._postgresql_conf, exclude)})
+
     def setup_server_parameters(self) -> None:
         self._server_parameters = self.get_server_parameters(self._config)
         self._adjust_recovery_parameters()
@@ -623,7 +631,24 @@ class ConfigHandler(object):
                                           'recovery_target_action', 'standby_mode', self._triggerfile_wrong_name})
         return CaseInsensitiveSet(self._RECOVERY_PARAMETERS - skip_params)
 
-    def _read_recovery_params(self) -> Tuple[Optional[CaseInsensitiveDict], Optional[bool]]:
+    def _read_recovery_params(self) -> Tuple[Optional[CaseInsensitiveDict], bool]:
+        """Read current recovery parameters values.
+
+        .. note::
+            We query Postgres only if we detected that Postgresql was restarted
+            or when at least one of the following files was updated:
+
+                * ``postgresql.conf``;
+                * ``postgresql.auto.conf``;
+                * ``passfile`` that is used in the ``primary_conninfo``.
+
+        :returns: a tuple with two elements:
+
+            * :class:`CaseInsensitiveDict` object with current values of recovery parameters,
+              or ``None`` if no configuration files were updated;
+
+            * ``True`` if new values of recovery parameters were queried, ``False`` otherwise.
+        """
         if self._postgresql.is_starting():
             return None, False
 
@@ -644,11 +669,20 @@ class ConfigHandler(object):
             self._postgresql_conf_mtime = pg_conf_mtime
             self._auto_conf_mtime = auto_conf_mtime
             self._postmaster_ctime = postmaster_ctime
-        except Exception:
+        except Exception as exc:
+            if all((isinstance(exc, PostgresConnectionException),
+                    self._postgresql_conf_mtime == pg_conf_mtime,
+                    self._auto_conf_mtime == auto_conf_mtime,
+                    self._passfile_mtime == passfile_mtime,
+                    self._postmaster_ctime != postmaster_ctime)):
+                # We detected that the connection to postgres fails, but the process creation time of the postmaster
+                # doesn't match the old value. It is an indicator that Postgres crashed and either doing crash
+                # recovery or down. In this case we return values like nothing changed in the config.
+                return None, False
             values = None
         return values, True
 
-    def _read_recovery_params_pre_v12(self) -> Tuple[Optional[CaseInsensitiveDict], Optional[bool]]:
+    def _read_recovery_params_pre_v12(self) -> Tuple[Optional[CaseInsensitiveDict], bool]:
         recovery_conf_mtime = mtime(self._recovery_conf)
         passfile_mtime = mtime(self._passfile) if self._passfile else False
         if recovery_conf_mtime == self._recovery_conf_mtime and passfile_mtime == self._passfile_mtime:
@@ -896,14 +930,15 @@ class ConfigHandler(object):
         listen_addresses, port = split_host_port(config['listen'], 5432)
         parameters.update(cluster_name=self._postgresql.scope, listen_addresses=listen_addresses, port=str(port))
         if not self._postgresql.global_config or self._postgresql.global_config.is_synchronous_mode:
-            if self._synchronous_standby_names is None:
+            synchronous_standby_names = self._server_parameters.get('synchronous_standby_names')
+            if synchronous_standby_names is None:
                 if self._postgresql.global_config and self._postgresql.global_config.is_synchronous_mode_strict\
                         and self._postgresql.role in ('master', 'primary', 'promoted'):
                     parameters['synchronous_standby_names'] = '*'
                 else:
                     parameters.pop('synchronous_standby_names', None)
             else:
-                parameters['synchronous_standby_names'] = self._synchronous_standby_names
+                parameters['synchronous_standby_names'] = synchronous_standby_names
 
         # Handle hot_standby <-> replica rename
         if parameters.get('wal_level') == ('hot_standby' if self._postgresql.major_version >= 90600 else 'replica'):
@@ -979,17 +1014,14 @@ class ConfigHandler(object):
         self._postgresql.connection_string = uri('postgres', netloc, self._postgresql.database)
         self._postgresql.set_connection_kwargs(self.local_connect_kwargs)
 
-    def _get_pg_settings(
+    def _get_pg_settings(self, names: Collection[str]) -> Dict[Any, Tuple[Any, ...]]:
-            self, names: Collection[str]
-    ) -> Dict[str, Tuple[str, str, Optional[str], str, str, Optional[str]]]:
         return {r[0]: r for r in self._postgresql.query(('SELECT name, setting, unit, vartype, context, sourcefile'
                                                          + ' FROM pg_catalog.pg_settings '
                                                          + ' WHERE pg_catalog.lower(name) = ANY(%s)'),
                                                         [n.lower() for n in names])}
 
     @staticmethod
-    def _handle_wal_buffers(old_values: Dict[str, Tuple[str, str, Optional[str], str, str, Optional[str]]],
+    def _handle_wal_buffers(old_values: Dict[Any, Tuple[Any, ...]], changes: CaseInsensitiveDict) -> None:
-                            changes: CaseInsensitiveDict) -> None:
         wal_block_size = parse_int(old_values['wal_block_size'][1]) or 8192
         wal_segment_size = old_values['wal_segment_size']
         wal_segment_unit = parse_int(wal_segment_size[2], 'B') or 8192 \
@@ -1106,12 +1138,11 @@ class ConfigHandler(object):
     def set_synchronous_standby_names(self, value: Optional[str]) -> Optional[bool]:
         """Updates synchronous_standby_names and reloads if necessary.
         :returns: True if value was updated."""
-        if value != self._synchronous_standby_names:
+        if value != self._server_parameters.get('synchronous_standby_names'):
             if value is None:
                 self._server_parameters.pop('synchronous_standby_names', None)
             else:
                 self._server_parameters['synchronous_standby_names'] = value
-            self._synchronous_standby_names = value
             if self._postgresql.state == 'running':
                 self.write_postgresql_conf()
                 self._postgresql.reload()

patroni/postgresql/rewind.py

@@ -158,7 +158,7 @@ class Rewind(object):
     def _get_local_timeline_lsn(self) -> Tuple[Optional[bool], Optional[int], Optional[int]]:
         if self._postgresql.is_running():  # if postgres is running - get timeline from replication connection
             in_recovery = True
-            timeline = self._postgresql.received_timeline() or self._postgresql.get_replica_timeline()
+            timeline = self._postgresql.get_replica_timeline()
             lsn = self._postgresql.replayed_location()
         else:  # otherwise analyze pg_controldata output
             in_recovery, timeline, lsn = self._get_local_timeline_lsn_from_controldata()

patroni/postgresql/slots.py

@@ -1,11 +1,15 @@
+"""Replication slot handling.
+
+Provides classes for the creation, monitoring, management and synchronisation of PostgreSQL replication slots.
+"""
+
 import logging
 import os
 import shutil
-
 from collections import defaultdict
 from contextlib import contextmanager
 from threading import Condition, Thread
-from typing import Any, Dict, Iterator, List, Optional, Union, Tuple, TYPE_CHECKING
+from typing import Any, Dict, Iterator, List, Optional, Union, Tuple, TYPE_CHECKING, Collection
 
 from .connection import get_connection_cursor
 from .misc import format_lsn, fsync_dir
@@ -43,9 +47,17 @@ def compare_slots(s1: Dict[str, Any], s2: Dict[str, Any], dbid: str = 'database'
 
 
 class SlotsAdvanceThread(Thread):
+    """Daemon process :class:``Thread`` object for advancing logical replication slots on replicas.
+
+    This ensures that slot advancing queries sent to postgres do not block the main loop.
+    """
 
     def __init__(self, slots_handler: 'SlotsHandler') -> None:
-        super(SlotsAdvanceThread, self).__init__()
+        """Create and start a new thread for handling slot advance queries.
+
+        :param slots_handler: The calling class instance for reference to slot information attributes.
+        """
+        super().__init__()
         self.daemon = True
         self._slots_handler = slots_handler
 
@@ -59,6 +71,13 @@ class SlotsAdvanceThread(Thread):
         self.start()
 
     def sync_slot(self, cur: Union['cursor', 'Cursor[Any]'], database: str, slot: str, lsn: int) -> None:
+        """Execute a ``pg_replication_slot_advance`` query and store success for scheduled synchronisation task.
+
+        :param cur: database connection cursor.
+        :param database: name of the database associated with the slot.
+        :param slot: name of the slot to be synchronised.
+        :param lsn: last known LSN position
+        """
         failed = copy = False
         try:
             cur.execute("SELECT pg_catalog.pg_replication_slot_advance(%s, %s)", (slot, format_lsn(lsn)))
@@ -80,6 +99,11 @@ class SlotsAdvanceThread(Thread):
                     self._scheduled.pop(database)
 
     def sync_slots_in_database(self, database: str, slots: List[str]) -> None:
+        """Synchronise slots for a single database.
+
+        :param database: name of the database.
+        :param slots: list of slot names to synchronise.
+        """
         with self._slots_handler.get_local_connection_cursor(dbname=database, options='-c statement_timeout=0') as cur:
             for slot in slots:
                 with self._condition:
@@ -88,6 +112,7 @@ class SlotsAdvanceThread(Thread):
                     self.sync_slot(cur, database, slot, lsn)
 
     def sync_slots(self) -> None:
+        """Synchronise slots for all scheduled databases."""
         with self._condition:
             databases = list(self._scheduled.keys())
         for database in databases:
@@ -100,6 +125,12 @@ class SlotsAdvanceThread(Thread):
                     logger.error('Failed to advance replication slots in database %s: %r', database, e)
 
     def run(self) -> None:
+        """Thread main loop entrypoint.
+
+        .. note::
+            Thread will wait until a sync is scheduled from outside, normally triggered during the HA loop or a wakeup
+            call.
+        """
         while True:
             with self._condition:
                 if not self._scheduled:
@@ -108,6 +139,14 @@ class SlotsAdvanceThread(Thread):
             self.sync_slots()
 
     def schedule(self, advance_slots: Dict[str, Dict[str, int]]) -> Tuple[bool, List[str]]:
+        """Trigger a synchronisation of slots.
+
+        This is the main entrypoint for Patroni HA loop wakeup call.
+
+        :param advance_slots: dictionary containing slots that need to be advanced
+
+        :return: tuple of failure status and a list of slots to be copied
+        """
         with self._condition:
             for database, values in advance_slots.items():
                 self._scheduled[database].update(values)
@@ -119,15 +158,27 @@ class SlotsAdvanceThread(Thread):
         return ret
 
     def on_promote(self) -> None:
+        """Reset state of the daemon."""
         with self._condition:
             self._scheduled.clear()
             self._failed = False
             self._copy_slots = []
 
 
-class SlotsHandler(object):
+class SlotsHandler:
+    """Handler for managing and storing information on replication slots in PostgreSQL.
+
+    :ivar pg_replslot_dir: system location path of the PostgreSQL replication slots.
+    :ivar _logical_slots_processing_queue: yet to be processed logical replication slots on the primary
+    """
 
     def __init__(self, postgresql: 'Postgresql') -> None:
+        """Create an instance with storage attributes for replication slots and schedule the first synchronisation.
+
+        :param postgresql: Calling class instance providing interface to PostgreSQL.
+        """
+        self._force_readiness_check = False
+        self._schedule_load_slots = False
         self._postgresql = postgresql
         self._advance = None
         self._replication_slots: Dict[str, Dict[str, Any]] = {}  # already existing replication slots
@@ -136,23 +187,46 @@ class SlotsHandler(object):
         self.schedule()
 
     def _query(self, sql: str, *params: Any) -> Union['cursor', 'Cursor[Any]']:
+        """Helper method for :meth:`Postgresql.query`.
+
+        :param sql: SQL statement to execute.
+        :param params: parameters to pass through to :meth:`Postgresql.query`.
+
+        :returns: query response.
+        """
         return self._postgresql.query(sql, *params, retry=False)
 
     @staticmethod
-    def _copy_items(src: Dict[str, Any], dst: Dict[str, Any], keys: Optional[List[str]] = None) -> None:
+    def _copy_items(src: Dict[str, Any], dst: Dict[str, Any], keys: Optional[Collection[str]] = None) -> None:
+        """Select values from *src* dictionary to update in *dst* dictionary for optional supplied *keys*.
+
+        :param src: source dictionary that *keys* will be looked up from.
+        :param dst: destination dictionary to be updated.
+        :param keys: optional list of keys to be looked up in the source dictionary.
+        """
         dst.update({key: src[key] for key in keys or ('datoid', 'catalog_xmin', 'confirmed_flush_lsn')})
 
     def process_permanent_slots(self, slots: List[Dict[str, Any]]) -> Dict[str, int]:
-        """This methods solves three problems at once (I know, it is weird).
+        """Process replication slot information from the host and prepare information used in subsequent cluster tasks.
+
+        .. note::
+            This methods solves three problems.
+
+            The ``cluster_info_query`` from :class:``Postgresql`` is executed every HA loop and returns information
+            about all replication slots that exists on the current host.
+
+            Based on this information perform the following actions:
+
+            1. For the primary we want to expose to DCS permanent logical slots, therefore build (and return) a dict
+               that maps permanent logical slot names to ``confirmed_flush_lsn``.
+            2. detect if one of the previously known permanent slots is missing and schedule resync.
+            3. Update the local cache with the fresh ``catalog_xmin`` and ``confirmed_flush_lsn`` for every known slot.
 
-        The cluster_info_query from `Postgresql` is executed every HA loop and returns
-        information about all replication slots that exists on the current host.
-        Based on this information we perform the following actions:
-        1. For the primary we want to expose to DCS permanent logical slots, therefore the method
-           builds (and returns) a dict, that maps permanent logical slot names and confirmed_flush_lsns.
-        2. This method also detects if one of the previously known permanent slots got missing and schedules resync.
-        3. Updates the local cache with the fresh catalog_xmin and confirmed_flush_lsn for every known slot.
            This info is used when performing the check of logical slot readiness on standbys.
+
+        :param slots: replication slot information that exists on the current host.
+
+        :return: dictionary of logical slot names to ``confirmed_flush_lsn``.
         """
         ret: Dict[str, int] = {}
 
@@ -174,13 +248,23 @@ class SlotsHandler(object):
         return ret
 
     def load_replication_slots(self) -> None:
+        """Query replication slot information from the database and store it for processing by other tasks.
+
+        .. note::
+            Only supported from PostgreSQL version 9.4 onwards.
+
+        Store replication slot ``name``, ``type``, ``plugin``, ``database`` and ``datoid``.
+        If PostgreSQL version is 10 or newer also store ``catalog_xmin`` and ``confirmed_flush_lsn``.
+
+        When using logical slots, store information separately for slot synchronisation  on replica nodes.
+        """
         if self._postgresql.major_version >= 90400 and self._schedule_load_slots:
             replication_slots: Dict[str, Dict[str, Any]] = {}
-            extra = ", catalog_xmin, pg_catalog.pg_wal_lsn_diff(confirmed_flush_lsn, '0/0')::bigint"\
+            extra = ", catalog_xmin, pg_catalog.pg_wal_lsn_diff(confirmed_flush_lsn, '0/0')::bigint" \
                 if self._postgresql.major_version >= 100000 else ""
             skip_temp_slots = ' WHERE NOT temporary' if self._postgresql.major_version >= 100000 else ''
-            cursor = self._query('SELECT slot_name, slot_type, plugin, database, datoid'
+            cursor = self._query(f'SELECT slot_name, slot_type, plugin, database, datoid'
-                                 '{0} FROM pg_catalog.pg_replication_slots{1}'.format(extra, skip_temp_slots))
+                                 f'{extra} FROM pg_catalog.pg_replication_slots{skip_temp_slots}')
             for r in cursor:
                 value = {'type': r[1]}
                 if r[1] == 'logical':
@@ -196,16 +280,34 @@ class SlotsHandler(object):
                 self._force_readiness_check = False
 
     def ignore_replication_slot(self, cluster: Cluster, name: str) -> bool:
+        """Check if slot *name* should not be managed by Patroni.
+
+        :param cluster: cluster state information object.
+        :param name: name of the slot to ignore
+
+        :returns: ``True`` if slot *name* matches any slot specified in ``ignore_slots`` configuration,
+                 otherwise will pass through and return result of :meth:`CitusHandler.ignore_replication_slot`.
+        """
         slot = self._replication_slots[name]
         if cluster.config:
             for matcher in cluster.config.ignore_slots_matchers:
-                if ((matcher.get("name") is None or matcher["name"] == name)
+                if (
-                   and all(not matcher.get(a) or matcher[a] == slot.get(a) for a in ('database', 'plugin', 'type'))):
+                        (matcher.get("name") is None or matcher["name"] == name)
+                        and all(not matcher.get(a) or matcher[a] == slot.get(a)
+                                for a in ('database', 'plugin', 'type'))
+                ):
                     return True
         return self._postgresql.citus_handler.ignore_replication_slot(slot)
 
     def drop_replication_slot(self, name: str) -> Tuple[bool, bool]:
-        """Returns a tuple(active, dropped)"""
+        """Drop a named slot from Postgres.
+
+        :param name: name of the slot to be dropped.
+
+        :returns: a tuple of ``active`` and ``dropped``. ``active`` is ``True`` if the slot is active,
+                  ``dropped`` is ``True`` if the slot was successfully dropped. If the slot was not found return
+                  ``False`` for both.
+        """
         cursor = self._query(('WITH slots AS (SELECT slot_name, active'
                               ' FROM pg_catalog.pg_replication_slots WHERE slot_name = %s),'
                               ' dropped AS (SELECT pg_catalog.pg_drop_replication_slot(slot_name),'
@@ -215,10 +317,22 @@ class SlotsHandler(object):
         row = cursor.fetchone()
         if not row:
             row = (False, False)
-        return row
+        return row[0], row[1]
 
     def _drop_incorrect_slots(self, cluster: Cluster, slots: Dict[str, Any], paused: bool) -> None:
-        # drop old replication slots which are not presented in desired slots
+        """Compare required slots and configured as permanent slots with those found, dropping extraneous ones.
+
+        .. note::
+            Slots that are not contained in *slots* will be dropped.
+            Slots can be filtered out with ``ignore_slots`` configuration.
+
+            Slots that have matching names but do not match attributes in *slots* will also be dropped.
+
+        :param cluster: cluster state information object.
+        :param slots: dictionary of desired slot names as keys with slot attributes as a dictionary value, if known.
+        :param paused: ``True`` if the patroni cluster is currently in a paused state.
+        """
+        # drop old replication slots which are not presented in desired slots.
         for name in set(self._replication_slots) - set(slots):
             if not paused and not self.ignore_replication_slot(cluster, name):
                 active, dropped = self.drop_replication_slot(name)
@@ -230,6 +344,8 @@ class SlotsHandler(object):
                         logger.debug("Unable to drop unknown replication slot '%s', slot is still active", name)
                     else:
                         logger.error("Failed to drop replication slot '%s'", name)
+
+        # drop slots with matching names but attributes that do not match, e.g. `plugin` or `database`.
         for name, value in slots.items():
             if name in self._replication_slots and not compare_slots(value, self._replication_slots[name]):
                 logger.info("Trying to drop replication slot '%s' because value is changing from %s to %s",
@@ -241,31 +357,57 @@ class SlotsHandler(object):
                     self._schedule_load_slots = True
 
     def _ensure_physical_slots(self, slots: Dict[str, Any]) -> None:
+        """Create any missing physical replication *slots*.
+
+        Any failures are logged and do not interrupt creation of all *slots*.
+
+        :param slots: A dictionary mapping slot name to slot attributes. This method only considers a slot
+                      if the value is a dictionary with the key ``type`` and a value of ``physical``.
+        """
         immediately_reserve = ', true' if self._postgresql.major_version >= 90600 else ''
         for name, value in slots.items():
             if name not in self._replication_slots and value['type'] == 'physical':
                 try:
-                    self._query(("SELECT pg_catalog.pg_create_physical_replication_slot(%s{0})"
+                    self._query(f"SELECT pg_catalog.pg_create_physical_replication_slot(%s{immediately_reserve})"
-                                 " WHERE NOT EXISTS (SELECT 1 FROM pg_catalog.pg_replication_slots"
+                                f" WHERE NOT EXISTS (SELECT 1 FROM pg_catalog.pg_replication_slots"
-                                 " WHERE slot_type = 'physical' AND slot_name = %s)").format(
+                                f" WHERE slot_type = 'physical' AND slot_name = %s)",
-                                     immediately_reserve), name, name)
+                                name, name)
                 except Exception:
                     logger.exception("Failed to create physical replication slot '%s'", name)
                 self._schedule_load_slots = True
 
     @contextmanager
     def get_local_connection_cursor(self, **kwargs: Any) -> Iterator[Union['cursor', 'Cursor[Any]']]:
+        """Create a new database connection to local server.
+
+        Create a non-blocking connection cursor to avoid the situation where an execution of the query of
+        ``pg_replication_slot_advance`` takes longer than the timeout on a HA loop, which could cause a false
+        failure state.
+
+        :param kwargs: Any keyword arguments to pass to :func:`psycopg.connect`.
+
+        :yields: connection cursor object, note implementation varies depending on version of :mod:`psycopg`.
+        """
         conn_kwargs = self._postgresql.config.local_connect_kwargs
         conn_kwargs.update(kwargs)
         with get_connection_cursor(**conn_kwargs) as cur:
             yield cur
 
     def _ensure_logical_slots_primary(self, slots: Dict[str, Any]) -> None:
+        """Create any missing logical replication *slots* on the primary.
+
+        If the logical slot already exists, copy state information into the replication slots structure stored in the
+        class instance.
+
+        :param slots: Slots that should exist are supplied in a dictionary, mapping slot name to any attributes.
+                      The method will only consider slots that have a value that is a dictionary with a key ``type``
+                      with a value that is ``logical``.
+
+        """
         # Group logical slots to be created by database name
         logical_slots: Dict[str, Dict[str, Dict[str, Any]]] = defaultdict(dict)
         for name, value in slots.items():
             if value['type'] == 'logical':
-                # If the logical already exists, copy some information about it into the original structure
                 if self._replication_slots.get(name, {}).get('datoid'):
                     self._copy_items(self._replication_slots[name], value)
                 else:
@@ -287,21 +429,45 @@ class SlotsHandler(object):
                     self._schedule_load_slots = True
 
     def schedule_advance_slots(self, slots: Dict[str, Dict[str, int]]) -> Tuple[bool, List[str]]:
+        """Wrapper to ensure slots advance daemon thread is started if not already.
+
+        :param slots: dictionary containing slot information.
+
+        :return: tuple with the result of the scheduling of slot advancement: ``failed`` and list of slots to copy.
+        """
         if not self._advance:
             self._advance = SlotsAdvanceThread(self)
         return self._advance.schedule(slots)
 
     def _ensure_logical_slots_replica(self, cluster: Cluster, slots: Dict[str, Any]) -> List[str]:
+        """Update logical *slots* on replicas.
+
+        If the logical slot already exists, copy state information into the replication slots structure stored in the
+        class instance. Slots that exist are also advanced if their ``confirmed_flush_lsn`` is greater than the stored
+        state of the slot.
+
+        As logical slots can only be created when the primary is available, pass the list of slots that need to be
+        copied back to the caller. They will be created on replicas with :meth:`SlotsHandler.copy_logical_slots`.
+
+        :param cluster: object containing stateful information for the cluster.
+        :param slots: A dictionary mapping slot name to slot attributes. This method only considers a slot
+                      if the value is a dictionary with the key ``type`` and a value of ``logical``.
+
+        :returns: list of slots to be copied from the primary.
+        """
         # Group logical slots to be advanced by database name
         advance_slots: Dict[str, Dict[str, int]] = defaultdict(dict)
-        create_slots: List[str] = []  # And collect logical slots to be created on the replica
+        create_slots: List[str] = []  # Collect logical slots to be created on the replica
+
         for name, value in slots.items():
-            if value['type'] == 'logical':
+            if value['type'] != 'logical':
+                continue
+
             # If the logical already exists, copy some information about it into the original structure
             if self._replication_slots.get(name, {}).get('datoid'):
                 self._copy_items(self._replication_slots[name], value)
                 if cluster.slots and name in cluster.slots:
-                        try:  # Skip slots that doesn't need to be advanced
+                    try:  # Skip slots that don't need to be advanced
                         if value['confirmed_flush_lsn'] < int(cluster.slots[name]):
                             advance_slots[value['database']][name] = int(cluster.slots[name])
                     except Exception as e:
@@ -309,6 +475,11 @@ class SlotsHandler(object):
             elif cluster.slots and name in cluster.slots:  # We want to copy only slots with feedback in a DCS
                 create_slots.append(name)
 
+        # Slots to be copied from the primary should be removed from the *slots* structure,
+        # otherwise Patroni falsely assumes that they already exist.
+        for name in create_slots:
+            slots.pop(name)
+
         error, copy_slots = self.schedule_advance_slots(advance_slots)
         if error:
             self._schedule_load_slots = True
@@ -316,13 +487,28 @@ class SlotsHandler(object):
 
     def sync_replication_slots(self, cluster: Cluster, nofailover: bool,
                                replicatefrom: Optional[str] = None, paused: bool = False) -> List[str]:
+        """During the HA loop read, check and alter replication slots found in the cluster.
+
+        Read physical and logical slots found on the primary, then compare to those configured in the DCS.
+        Drop any slots that do not match those required by configuration and are not configured as permanent.
+        Create any missing physical slots. If we are the leader then logical slots too, otherwise if logical slots
+        are known and active create them on replica nodes.
+
+        :param cluster: object containing stateful information for the cluster.
+        :param nofailover: ``True`` if this node has been tagged to not be a failover candidate.
+        :param replicatefrom: the tag containing the node to replicate from.
+        :param paused: ``True`` if the cluster is in maintenance mode.
+
+        :returns: list of logical replication slots names that should be copied from the primary.
+        """
         ret = []
-        if self._postgresql.major_version >= 90400 and cluster.config:
+        if self._postgresql.major_version >= 90400 and self._postgresql.global_config and cluster.config:
             try:
                 self.load_replication_slots()
 
-                slots = cluster.get_replication_slots(self._postgresql.name, self._postgresql.role,
+                slots = cluster.get_replication_slots(
-                                                      nofailover, self._postgresql.major_version, True)
+                    self._postgresql.name, self._postgresql.role, nofailover, self._postgresql.major_version,
+                    is_standby_cluster=self._postgresql.global_config.is_standby_cluster, show_error=True)
 
                 self._drop_incorrect_slots(cluster, slots, paused)
 
@@ -344,6 +530,16 @@ class SlotsHandler(object):
 
     @contextmanager
     def _get_leader_connection_cursor(self, leader: Leader) -> Iterator[Union['cursor', 'Cursor[Any]']]:
+        """Create a new database connection to the leader.
+
+        .. note::
+            Uses rewind user credentials because it has enough permissions to read files from PGDATA.
+            Sets the options ``connect_timeout`` to ``3`` and ``statement_timeout`` to ``2000``.
+
+        :param leader: object with information on the leader
+
+        :yields: connection cursor object, note implementation varies depending on version of ``psycopg``.
+        """
         conn_kwargs = leader.conn_kwargs(self._postgresql.config.rewind_credentials)
         conn_kwargs['dbname'] = self._postgresql.database
         with get_connection_cursor(connect_timeout=3, options="-c statement_timeout=2000", **conn_kwargs) as cur:
@@ -446,6 +642,11 @@ class SlotsHandler(object):
                     logger.info('Logical slot %s is safe to be used after a failover', name)
 
     def copy_logical_slots(self, cluster: Cluster, create_slots: List[str]) -> None:
+        """Create logical replication slots on standby nodes.
+
+        :param cluster: object containing stateful information for the cluster.
+        :param create_slots: list of slot names to copy from the primary.
+        """
         leader = cluster.leader
         if not leader:
             return
@@ -497,11 +698,23 @@ class SlotsHandler(object):
             self._postgresql.start()
 
     def schedule(self, value: Optional[bool] = None) -> None:
+        """Schedule the loading of slot information from the database.
+
+        :param value: the optional value can be used to unschedule if set to ``False`` or force it to be ``True``.
+                      If it is omitted the value will be ``True`` if this PostgreSQL node supports slot replication.
+        """
         if value is None:
             value = self._postgresql.major_version >= 90400
         self._schedule_load_slots = self._force_readiness_check = value
 
     def on_promote(self) -> None:
+        """Entry point from HA cycle used when a standby node is to be promoted to primary.
+
+        .. note::
+            If logical replication slot synchronisation is enabled then slot advancement will be triggered.
+            If any logical slots that were copied are yet to be confirmed as ready a warning message will be logged.
+
+        """
         if self._advance:
             self._advance.on_promote()
 

patroni/postgresql/sync.py

@@ -283,12 +283,13 @@ END;$$""")
                 self._ready_replicas[replica.application_name] = replica.pid
 
     def current_state(self, cluster: Cluster) -> Tuple[CaseInsensitiveSet, CaseInsensitiveSet]:
-        """Finds best candidates to be the synchronous standbys.
+        """Find the best candidates to be the synchronous standbys.
 
         Current synchronous standby is always preferred, unless it has disconnected or does not want to be a
         synchronous standby any longer.
 
         Standbys are selected based on values from the global configuration:
+
         - `maximum_lag_on_syncnode`: would help swapping unhealthy sync replica in case if it stops
           responding (or hung). Please set the value high enough so it won't unncessarily swap sync
           standbys during high loads. Any value less or equal of 0 keeps the behavior backward compatible.

patroni/postgresql/validator.py

@@ -178,10 +178,11 @@ class ValidatorFactory:
 
         :returns: the Patroni validator object that corresponds to the specification found in *validator*.
 
-        :raises :class:`ValidatorFactoryNoType`: if *validator* contains no ``type`` key.
+        :raises:
-        :raises :class:`ValidatorFactoryInvalidType`: if ``type`` key from *validator* contains an invalid value.
+            :class:`ValidatorFactoryNoType`: if *validator* contains no ``type`` key.
-        :raises :class:`ValidatorFactoryInvalidSpec`: if *validator* contains an invalid set of attributes for the
+            :class:`ValidatorFactoryInvalidType`: if ``type`` key from *validator* contains an invalid value.
-            given ``type``.
+            :class:`ValidatorFactoryInvalidSpec`: if *validator* contains an invalid set of attributes for the given
+                ``type``.
 
         :Example:
 
@@ -265,7 +266,8 @@ def _read_postgres_gucs_validators_file(file: str) -> Dict[str, Any]:
     :returns: the YAML content parsed into a Python object. If any issue is faced while reading/parsing the file, then
         return ``None``.
 
-    :raises :class:`InvalidGucValidatorsFile`: if faces an issue while reading or parsing *file*.
+    :raises:
+        :class:`InvalidGucValidatorsFile`: if faces an issue while reading or parsing *file*.
     """
     try:
         with open(file, encoding='UTF-8') as stream:
@@ -462,11 +464,13 @@ def transform_postgresql_parameter_value(version: int, name: str, value: Any,
     :param value: value of the Postgres GUC.
     :param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres
         GUC. Used for a couple purposes:
-        * Disallow writing GUCs to ``postgresql.conf`` that does not exist in Postgres *version*;
-        * Avoid ignoring GUC *name* if it does not have a validator in ``parameters``, but is a valid GUC in Postgres
-            *version*.
 
-    :returns: The return value may be one among
+            * Disallow writing GUCs to ``postgresql.conf`` that does not exist in Postgres *version*;
+            * Avoid ignoring GUC *name* if it does not have a validator in ``parameters``, but is a valid GUC in
+                Postgres *version*.
+
+    :returns: The return value may be one among:
+
         * The original *value* if *name* seems to be an extension GUC (contains a period '.'); or
         * ``None`` if **name** is a recovery GUC; or
         * *value* transformed to the expected format for GUC *name* in Postgres *version* using validators defined in
@@ -490,10 +494,11 @@ def transform_recovery_parameter_value(version: int, name: str, value: Any,
     :param value: value of the Postgres recovery GUC.
     :param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres
         GUC. Used for a couple purposes:
+
             * Disallow writing GUCs to ``recovery.conf`` (or ``postgresql.conf`` depending on *version*), that does not
                 exist in Postgres *version*;
-        * Avoid ignoring recovery GUC *name* if it does not have a validator in ``recovery_parameters``, but is a valid
+            * Avoid ignoring recovery GUC *name* if it does not have a validator in ``recovery_parameters``, but is a
-            GUC in Postgres *version*.
+                valid GUC in Postgres *version*.
 
     :returns: *value* transformed to the expected format for recovery GUC *name* in Postgres *version* using validators
         defined in ``recovery_parameters``. It can also return ``None``. See :func:`_transform_parameter_value`.

patroni/psycopg.py

@@ -1,7 +1,8 @@
-"""Abstraction layer for ``psycopg`` module.
+"""Abstraction layer for :mod:`psycopg` module.
 
-This module is able to handle both ``pyscopg2`` and ``psycopg3``, and it exposes a common interface for both.
+This module is able to handle both :mod:`pyscopg2` and :mod:`psycopg`, and it exposes a common interface for both.
-``psycopg2`` takes precedence. ``psycopg3`` will only be used if ``psycopg2`` is either absent or older than ``2.5.4``.
+:mod:`psycopg2` takes precedence. :mod:`psycopg` will only be used if :mod:`psycopg2` is either absent or older than
+``2.5.4``.
 """
 from typing import Any, Optional, TYPE_CHECKING, Union
 if TYPE_CHECKING:  # pragma: no cover
@@ -28,7 +29,7 @@ try:
         """Quote *value* as a SQL literal.
 
         .. note::
-            *value* is quoted through ``psycopg`` adapters.
+            *value* is quoted through :mod:`psycopg2` adapters.
 
         :param value: value to be quoted.
         :param conn: if a connection is given then :func:`quote_literal` checks if any special handling based on server
@@ -44,14 +45,14 @@ except ImportError:
     from psycopg import connect as __connect, sql, Error, DatabaseError, OperationalError, ProgrammingError
 
     def _connect(dsn: Optional[str] = None, **kwargs: Any) -> 'Connection[Any]':
-        """Call ``psycopg.connect`` with ``dsn`` and ``**kwargs``.
+        """Call :func:`psycopg.connect` with *dsn* and ``**kwargs``.
 
         .. note::
             Will create ``server_version`` attribute in the returning connection, so it keeps compatibility with the
-            object that would be returned by ``psycopg2.connect``.
+            object that would be returned by :func:`psycopg2.connect`.
 
-        :param dsn: DSN to call ``psycopg.connect`` with.
+        :param dsn: DSN to call :func:`psycopg.connect` with.
-        :param kwargs: keyword arguments to call ``psycopg.connect`` with.
+        :param kwargs: keyword arguments to call :func:`psycopg.connect` with.
 
         :returns: a connection to the database.
         """
@@ -89,11 +90,11 @@ def connect(*args: Any, **kwargs: Any) -> Union['connection', 'Connection[Any]']
         It also enforces ``search_path=pg_catalog`` for non-replication connections to mitigate security issues as
         Patroni relies on superuser connections.
 
-    :param args: positional arguments to call ``connect`` function from ``psycopg`` module.
+    :param args: positional arguments to call :func:`~psycopg.connect` function from :mod:`psycopg` module.
-    :param kwargs: keyword arguments to call ``connect`` function from ``psycopg`` module.
+    :param kwargs: keyword arguments to call :func:`~psycopg.connect` function from :mod:`psycopg` module.
 
-    :returns: a connection to the database. Can be either a :class:`psycopg.Connection` if using ``psycopg3``, or a
+    :returns: a connection to the database. Can be either a :class:`psycopg.Connection` if using :mod:`psycopg`, or a
-        :class:`psycopg2.extensions.connection` if using ``psycopg2``.
+        :class:`psycopg2.extensions.connection` if using :mod:`psycopg2`.
     """
     if kwargs and 'replication' not in kwargs and kwargs.get('fallback_application_name') != 'Patroni ctl':
         options = [kwargs['options']] if 'options' in kwargs else []
@@ -109,7 +110,7 @@ def quote_ident(value: Any, conn: Optional[Union['cursor', 'connection', 'Connec
 
     :param value: value to be quoted.
     :param conn: connection to evaluate the returning string into. Can be either a :class:`psycopg.Connection` if
-        using ``psycopg3``, or a :class:`psycopg2.extensions.connection` if using ``psycopg2``.
+        using :mod:`psycopg`, or a :class:`psycopg2.extensions.connection` if using :mod:`psycopg2`.
 
     :returns: *value* quoted as a SQL identifier.
     """

patroni/request.py

@@ -34,10 +34,11 @@ class PatroniRequest(object):
         """Create a new :class:`PatroniRequest` instance with given *config*.
 
         :param config: Patroni YAML configuration.
-        :param insecure: how to deal with SSL certs verification
+        :param insecure: how to deal with SSL certs verification:
+
             * If ``True`` it will perform REST API requests without verifying SSL certs; or
             * If ``False`` it will perform REST API requests and verify SSL certs; or
-            * If ``None`` it will behave according to the value of ``ctl -> insecure`` configuration; or
+            * If ``None`` it will behave according to the value of ``ctl.insecure`` configuration; or
             * If none of the above applies, then it falls back to ``False``.
         """
         self._insecure = insecure
@@ -51,7 +52,7 @@ class PatroniRequest(object):
         :param config: Patroni YAML configuration.
         :param name: name of the setting value to be retrieved.
 
-        :returns: value of ``ctl -> *name*`` if present, ``None`` otherwise.
+        :returns: value of ``ctl.*name*`` if present, ``None`` otherwise.
         """
         return config.get('ctl', {}).get(name, default)
 
@@ -83,12 +84,13 @@ class PatroniRequest(object):
 
         :param config: Patroni YAML configuration.
         :param name: prefix of the Patroni SSL related setting name. Currently, supports these:
+
             * ``cert``: gets translated to ``certfile``
             * ``key``: gets translated to ``keyfile``
 
             Will attempt to fetch the requested key first from ``ctl`` section.
 
-        :returns: value of ``ctl -> *name*file`` if present, ``None`` otherwise.
+        :returns: value of ``ctl.*name*file`` if present, ``None`` otherwise.
         """
         value = self._get_ctl_value(config, name + 'file')
         self._apply_pool_param(name + '_file', value)
@@ -100,12 +102,12 @@ class PatroniRequest(object):
         Configure these HTTP headers for requests:
 
             * ``authorization``: based on Patroni' CTL or REST API authentication config;
-        * ``user-agent``: based on `patroni.utils.USER_AGENT`.
+            * ``user-agent``: based on ``patroni.utils.USER_AGENT``.
 
         Also configure SSL related settings for requests:
 
-        * ``ca_certs`` is configured if ``ctl -> cacert`` or ``restapi -> cafile`` is available;
+            * ``ca_certs`` is configured if ``ctl.cacert`` or ``restapi.cafile`` is available;
-        * ``cert``, ``key`` and ``key_password`` are configured if ``ctl -> certfile`` is available.
+            * ``cert``, ``key`` and ``key_password`` are configured if ``ctl.certfile`` is available.
 
         :param config: Patroni YAML configuration.
         """

patroni/utils.py

@@ -245,12 +245,14 @@ def convert_to_base_unit(value: Union[int, float], unit: str, base_unit: Optiona
     """Convert *value* as a *unit* of compute information or time to *base_unit*.
 
     :param value: value to be converted to the base unit.
-    :param unit: unit of *value*. Accepts these units (case sensitive)
+    :param unit: unit of *value*. Accepts these units (case sensitive):
+
             * For space: ``B``, ``kB``, ``MB``, ``GB``, or ``TB``;
             * For time: ``d``, ``h``, ``min``, ``s``, ``ms``, or ``us``.
 
     :param base_unit: target unit in the conversion. May contain the target unit with an associated value, e.g
-        ``512MB``. Accepts these units (case sensitive)
+        ``512MB``. Accepts these units (case sensitive):
+
             * For space: ``B``, ``kB``, or ``MB``;
             * For time: ``ms``, ``s``, or ``min``.
 
@@ -402,7 +404,8 @@ def compare_values(vartype: str, unit: Optional[str], old_value: Any, new_value:
     """Check if *old_value* and *new_value* are equivalent after parsing them as *vartype*.
 
     :param vartpe: the target type to parse *old_value* and *new_value* before comparing them. Accepts any among of the
-        following (case sensitive)
+        following (case sensitive):
+
         * ``bool``: parse values using :func:`parse_bool`; or
         * ``integer``: parse values using :func:`parse_int`; or
         * ``real``: parse values using :func:`parse_real`; or
@@ -459,7 +462,7 @@ def compare_values(vartype: str, unit: Optional[str], old_value: Any, new_value:
 
 
 def _sleep(interval: Union[int, float]) -> None:
-    """Wrap :func:`time.sleep`.
+    """Wrap :func:`~time.sleep`.
 
     :param interval: Delay execution for a given number of seconds. The argument may be a floating point number for
         subsecond precision.
@@ -536,6 +539,7 @@ class Retry(object):
         """Set next cycle delay.
 
         It will be the minimum value between:
+
             * current delay with ``backoff``; or
             * ``max_delay``.
         """
@@ -549,10 +553,14 @@ class Retry(object):
     def ensure_deadline(self, timeout: float, raise_ex: Optional[Exception] = None) -> bool:
         """Calculates, sets, and checks the remaining deadline time.
 
-        :param timeout: if the *deadline* is smaller than the provided *timeout* value raise *raise_ex* exception
+        :param timeout: if the *deadline* is smaller than the provided *timeout* value raise *raise_ex* exception.
-        :param raise_ex: the exception object that will be raised if the *deadline* is smaller than provided *timeout*
+        :param raise_ex: the exception object that will be raised if the *deadline* is smaller than provided *timeout*.
-        :returns: `False` if *deadline* is smaller than a provided *timeout* and *raise_ex* isn't set. Otherwise `True`
+
-        :raises Exception: if calculated deadline is smaller than provided *timeout*
+        :returns: ``False`` if *deadline* is smaller than a provided *timeout* and *raise_ex* isn't set. Otherwise
+            ``True``.
+
+        :raises:
+            :class:`Exception`: *raise_ex* if calculated deadline is smaller than provided *timeout*.
         """
         self.deadline = self.stoptime - time.time()
         if self.deadline < timeout:
@@ -565,6 +573,7 @@ class Retry(object):
         """Call a function *func* with arguments ``*args`` and ``*kwargs`` in a loop.
 
         *func* will be called until one of the following conditions is met:
+
             * It completes without throwing one of the configured ``retry_exceptions``; or
             * ``max_retries`` is exceeded.; or
             * ``deadline`` is exceeded.
@@ -576,7 +585,8 @@ class Retry(object):
         :param func: function to call.
         :param args: positional arguments to call *func* with.
         :params kwargs: keyword arguments to call *func* with.
-        :raises :class:`RetryFailedError`
+        :raises:
+            :class:`RetryFailedError`:
                 * If ``max_tries`` is exceeded; or
                 * If ``deadline`` is exceeded.
         """
@@ -613,7 +623,8 @@ def polling_loop(timeout: Union[int, float], interval: Union[int, float] = 1) ->
 
     :param timeout: for how long (in seconds) from now it should keep returning values.
     :param interval: for how long to sleep before returning a new value.
-    :rtype: Iterator[:class:`int`] with current iteration counter, starting from ``0``.
+
+    :yields: current iteration counter, starting from ``0``.
     """
     start_time = time.time()
     iteration = 0
@@ -627,11 +638,13 @@ def polling_loop(timeout: Union[int, float], interval: Union[int, float] = 1) ->
 def split_host_port(value: str, default_port: Optional[int]) -> Tuple[str, int]:
     """Extract host(s) and port from *value*.
 
-    :param value: string from where host(s) and port will be extracted. Accepts either of these formats
+    :param value: string from where host(s) and port will be extracted. Accepts either of these formats:
+
             * ``host:port``; or
             * ``host1,host2,...,hostn:port``.
 
         Each ``host`` portion of *value* can be either:
+
             * A FQDN; or
             * An IPv4 address; or
             * An IPv6 address, with or without square brackets.
@@ -669,15 +682,20 @@ def uri(proto: str, netloc: Union[List[str], Tuple[str, Union[int, str]], str],
 
     :param proto: the URI protocol.
     :param netloc: the URI host(s) and port. Can be specified in either way among
+
         * A :class:`list` or :class:`tuple`. The second item should be a port, and the first item should be composed of
             hosts in either of these formats:
+
             * ``host``; or.
             * ``host1,host2,...,hostn``.
+
         * A :class:`str` in either of these formats:
+
             * ``host:port``; or
             * ``host1,host2,...,hostn:port``.
 
         In all cases, each ``host`` portion of *netloc* can be either:
+
             * An FQDN; or
             * An IPv4 address; or
             * An IPv6 address, with or without square brackets.
@@ -698,10 +716,11 @@ def uri(proto: str, netloc: Union[List[str], Tuple[str, Union[int, str]], str],
 
 
 def iter_response_objects(response: HTTPResponse) -> Iterator[Dict[str, Any]]:
-    """Iterate over the chunks of a :class:`HTTPResponse` and yield each JSON document that is found along the way.
+    """Iterate over the chunks of a :class:`~urllib3.response.HTTPResponse` and yield each JSON document that is found.
 
     :param response: the HTTP response from which JSON documents will be retrieved.
-    :rtype: Iterator[:class:`dict`] with current JSON document.
+
+    :yields: current JSON document.
     """
     prev = ''
     decoder = JSONDecoder()
@@ -730,8 +749,8 @@ def iter_response_objects(response: HTTPResponse) -> Iterator[Dict[str, Any]]:
 def cluster_as_json(cluster: 'Cluster', global_config: Optional['GlobalConfig'] = None) -> Dict[str, Any]:
     """Get a JSON representation of *cluster*.
 
-    :param cluster: the :class:`Cluster` object to be parsed as JSON.
+    :param cluster: the :class:`~patroni.dcs.Cluster` object to be parsed as JSON.
-    :param global_config: optional :class:`GlobalConfig` object to check the cluster state.
+    :param global_config: optional :class:`~patroni.config.GlobalConfig` object to check the cluster state.
                           if not provided will be instantiated from the `Cluster.config`.
 
     :returns: JSON representation of *cluster*.
@@ -739,6 +758,7 @@ def cluster_as_json(cluster: 'Cluster', global_config: Optional['GlobalConfig']
     These are the possible keys in the returning object depending on the available information in *cluster*:
 
         * ``members``: list of members in the cluster. Each value is a :class:`dict` that may have the following keys:
+
             * ``name``: the name of the host (unique in the cluster). The ``members`` list is sorted by this key;
             * ``role``: ``leader``, ``standby_leader``, ``sync_standby``, or ``replica``;
             * ``state``: ``stopping``, ``stopped``, ``stop failed``, ``crashed``, ``running``, ``starting``,
@@ -752,8 +772,10 @@ def cluster_as_json(cluster: 'Cluster', global_config: Optional['GlobalConfig']
             * ``scheduled_restart``: scheduled restart timestamp, if any;
             * ``tags``: any tags that were set for this member;
             * ``lag``: replication lag, if applicable;
+
         * ``pause``: ``True`` if cluster is in maintenance mode;
         * ``scheduled_switchover``: if a switchover has been scheduled, then it contains this entry with these keys:
+
             * ``at``: timestamp when switchover was scheduled to occur;
             * ``from``: name of the member to be demoted;
             * ``to``: name of the member to be promoted.
@@ -832,12 +854,15 @@ def validate_directory(d: str, msg: str = "{} {}") -> None:
         If the directory does not exist, :func:`validate_directory` will attempt to create it.
 
     :param d: the directory to be checked.
-    :param msg: a message to be thrown when raising :class:`PatroniException`, if any issue is faced. It must contain
+    :param msg: a message to be thrown when raising :class:`~patroni.exceptions.PatroniException`, if any issue is
-        2 placeholders to be used by :func:`format`:
+        faced. It must contain 2 placeholders to be used by :func:`format`:
+
             * The first placeholder will be replaced with path *d*;
             * The second placeholder will be replaced with the error condition.
 
-    :raises :class:`PatroniException`: if any issue is observed while validating *d*. Can be thrown in these situations
+    :raises:
+        :class:`~patroni.exceptions.PatroniException`: if any issue is observed while validating *d*. Can be thrown if:
+
             * *d* did not exist, and :func:`validate_directory` was not able to create it; or
             * *d* is an existing directory, but Patroni is not able to write to that directory; or
             * *d* is an existing file, not a directory.
@@ -895,13 +920,22 @@ def keepalive_socket_options(timeout: int, idle: int, cnt: int = 3) -> Iterator[
     :param idle: value for ``TCP_KEEPIDLE``.
     :param cnt: value for ``TCP_KEEPCNT``.
 
-    :rtype: Iterator[Tuple[:class:`int`, :class:`int`, :class:`int`]] of all keepalive related socket options to be
+    :yields: all keepalive related socket options to be set. The first item in the tuple is the protocol, the second
-        set. The first item in the tuple is the protocol, the second item is the option, and the third item is the
+        item is the option, and the third item is the value to be used. The return values depend on the platform:
-        value to be used. The return values depend on the platform:
+
-        * ``Windows``: yield ``SO_KEEPALIVE``;
+            * ``Windows``:
-        * ``Linux``: yield ``SO_KEEPALIVE``, ``TCP_USER_TIMEOUT``, ``TCP_KEEPIDLE`, ``TCP_KEEPINTVL``, and
+                * ``SO_KEEPALIVE``.
-            ``TCP_KEEPCNT``;
+            * ``Linux``:
-        * ``MacOS``: yield ``SO_KEEPALIVE``, ``TCP_KEEPIDLE`, ``TCP_KEEPINTVL``, and ``TCP_KEEPCNT``
+                * ``SO_KEEPALIVE``;
+                * ``TCP_USER_TIMEOUT``;
+                * ``TCP_KEEPIDLE``;
+                * ``TCP_KEEPINTVL``;
+                * ``TCP_KEEPCNT``.
+            * ``MacOS``:
+                * ``SO_KEEPALIVE``;
+                * ``TCP_KEEPIDLE``;
+                * ``TCP_KEEPINTVL``;
+                * ``TCP_KEEPCNT``.
     """
     yield (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
 
@@ -939,7 +973,7 @@ def enable_keepalive(sock: socket.socket, timeout: int, idle: int, cnt: int = 3)
     :param idle: value for ``TCP_KEEPIDLE``.
     :param cnt: value for ``TCP_KEEPCNT``.
 
-    :returns: output of :func:`socket.ioctl` if we are on Windows, nothing otherwise.
+    :returns: output of :func:`~socket.ioctl` if we are on Windows, nothing otherwise.
     """
     SIO_KEEPALIVE_VALS = getattr(socket, 'SIO_KEEPALIVE_VALS', None)
     if SIO_KEEPALIVE_VALS is not None:  # Windows
@@ -953,23 +987,27 @@ def enable_keepalive(sock: socket.socket, timeout: int, idle: int, cnt: int = 3)
 def unquote(string: str) -> str:
     """Unquote a fully quoted *string*.
 
+    :param string: The string to be checked for quoting.
+
+    :returns: The string with quotes removed, if it is a fully quoted single string, or the original string if quoting
+        is not detected, or unquoting was not possible.
+
     :Examples:
 
         A *string* with quotes will have those quotes removed
+
         >>> unquote('"a quoted string"')
         'a quoted string'
 
         A *string* with multiple quotes will be returned as is
+
         >>> unquote('"a multi" "quoted string"')
         '"a multi" "quoted string"'
 
         So will a *string* with unbalanced quotes
+
         >>> unquote('unbalanced "quoted string')
         'unbalanced "quoted string'
-
-    :param string: The string to be checked for quoting.
-    :returns: The string with quotes removed, if it is a fully quoted single string,
-              or the original string if quoting is not detected, or unquoting was not possible.
     """
     try:
         ret = split(string)

patroni/validator.py

@@ -3,7 +3,7 @@
 
 This module contains facilities for validating configuration of Patroni processes.
 
-:var schema: configuration schema of the daemon launched by `patroni` command.
+:var schema: configuration schema of the daemon launched by ``patroni`` command.
 """
 import os
 import re
@@ -23,6 +23,7 @@ def data_directory_empty(data_dir: str) -> bool:
     """Check if PostgreSQL data directory is empty.
 
     :param data_dir: path to the PostgreSQL data directory to be checked.
+
     :returns: ``True`` if the data directory is empty.
     """
     if os.path.isfile(os.path.join(data_dir, "global", "pg_control")):
@@ -33,10 +34,12 @@ def data_directory_empty(data_dir: str) -> bool:
 def validate_connect_address(address: str) -> bool:
     """Check if options related to connection address were properly configured.
 
-    :param address: address to be validated in the format
+    :param address: address to be validated in the format ``host:ip``.
-        ``host:ip``.
+
     :returns: ``True`` if the address is valid.
-    :raises :class:`patroni.exceptions.ConfigParseError`:
+
+    :raises:
+        :class:`~patroni.exceptions.ConfigParseError`:
             * If the address is not in the expected format; or
             * If the host is set to not allowed values (``127.0.0.1``, ``0.0.0.0``, ``*``, ``::1``, or ``localhost``).
     """
@@ -52,20 +55,25 @@ def validate_connect_address(address: str) -> bool:
 def validate_host_port(host_port: str, listen: bool = False, multiple_hosts: bool = False) -> bool:
     """Check if host(s) and port are valid and available for usage.
 
-    :param host_port: the host(s) and port to be validated. It can be in either of these formats
+    :param host_port: the host(s) and port to be validated. It can be in either of these formats:
+
         * ``host:ip``, if *multiple_hosts* is ``False``; or
         * ``host_1,host_2,...,host_n:port``, if *multiple_hosts* is ``True``.
 
     :param listen: if the address is expected to be available for binding. ``False`` means it expects to connect to that
         address, and ``True`` that it expects to bind to that address.
     :param multiple_hosts: if *host_port* can contain multiple hosts.
+
     :returns: ``True`` if the host(s) and port are valid.
-    :raises: :class:`patroni.exceptions.ConfigParserError`:
+
+    :raises:
+        :class:`~patroni.exceptions.ConfigParseError`:
             * If the *host_port* is not in the expected format; or
             * If ``*`` was specified along with more hosts in *host_port*; or
             * If we are expecting to bind to an address that is already in use; or
             * If we are not able to connect to an address that we are expecting to do so; or
-        * If :class:`socket.gaierror` is thrown by socket module when attempting to connect to the given address(es).
+            * If :class:`~socket.gaierror` is thrown by socket module when attempting to connect to the given
+              address(es).
     """
     try:
         hosts, port = split_host_port(host_port, 1)
@@ -105,6 +113,7 @@ def validate_host_port_list(value: List[str]) -> bool:
     Call :func:`validate_host_port` with each item in *value*.
 
     :param value: list of host(s) and port items to be validated.
+
     :returns: ``True`` if all items are valid.
     """
     assert all([validate_host_port(v) for v in value]), "didn't pass the validation"
@@ -117,6 +126,7 @@ def comma_separated_host_port(string: str) -> bool:
     Call :func:`validate_host_port_list` with a list represented by the CSV *string*.
 
     :param string: comma-separated list of host and port items.
+
     :returns: ``True`` if all items in the CSV string are valid.
     """
     return validate_host_port_list([s.strip() for s in string.split(",")])
@@ -128,7 +138,7 @@ def validate_host_port_listen(host_port: str) -> bool:
     Call :func:`validate_host_port` with *listen* set to ``True``.
 
     :param host_port: the host and port to be validated. Must be in the format
-        `host:ip`.
+        ``host:ip``.
 
     :returns: ``True`` if the host and port are valid and available for binding.
     """
@@ -141,8 +151,9 @@ def validate_host_port_listen_multiple_hosts(host_port: str) -> bool:
     Call :func:`validate_host_port` with both *listen* and *multiple_hosts* set to ``True``.
 
     :param host_port: the host(s) and port to be validated. It can be in either of these formats
-        * `host:ip`; or
+
-        * `host_1,host_2,...,host_n:port`
+        * ``host:ip``; or
+        * ``host_1,host_2,...,host_n:port``
 
     :returns: ``True`` if the host(s) and port are valid and available for binding.
     """
@@ -153,8 +164,11 @@ def is_ipv4_address(ip: str) -> bool:
     """Check if *ip* is a valid IPv4 address.
 
     :param ip: the IP to be checked.
+
     :returns: ``True`` if the IP is an IPv4 address.
-    :raises :class:`patroni.exceptions.ConfigParserError`: if *ip* is not a valid IPv4 address.
+
+    :raises:
+        :class:`~patroni.exceptions.ConfigParseError`: if *ip* is not a valid IPv4 address.
     """
     try:
         socket.inet_aton(ip)
@@ -167,8 +181,11 @@ def is_ipv6_address(ip: str) -> bool:
     """Check if *ip* is a valid IPv6 address.
 
     :param ip: the IP to be checked.
+
     :returns: ``True`` if the IP is an IPv6 address.
-    :raises :class:`patroni.exceptions.ConfigParserError`: if *ip* is not a valid IPv6 address.
+
+    :raises:
+        :class:`~patroni.exceptions.ConfigParseError`: if *ip* is not a valid IPv6 address.
     """
     try:
         socket.inet_pton(socket.AF_INET6, ip)
@@ -222,8 +239,11 @@ def validate_data_dir(data_dir: str) -> bool:
         * Point to a non-empty directory that seems to contain a valid PostgreSQL data directory.
 
     :param data_dir: the value of ``postgresql.data_dir`` configuration option.
+
     :returns: ``True`` if the PostgreSQL data directory is valid.
-    :raises :class:`patroni.exceptions.ConfigParserError`:
+
+    :raises:
+        :class:`~patroni.exceptions.ConfigParseError`:
             * If no *data_dir* was given; or
             * If *data_dir* is a file and not a directory; or
             * If *data_dir* is a non-empty directory and:
@@ -270,7 +290,8 @@ def validate_binary_name(bin_name: str) -> bool:
 
     :returns: ``True`` if the conditions are true
 
-    :raises :class:`patroni.exceptions.ConfigParserError`: if:
+    :raises:
+        :class:`~patroni.exceptions.ConfigParseError` if:
             * *bin_name* is not set; or
             * the path join of the ``postgresql.bin_dir`` plus *bin_name* does not exist; or
             * the path join as above is not executable; or
@@ -301,7 +322,7 @@ class Result(object):
 
         .. note::
 
-            ``error`` attribute is only set if ``status`` is failed.
+            ``error`` attribute is only set if *status* is failed.
 
         :param status: if the validation succeeded.
         :param error: error message related to the validation that was performed, if the validation failed.
@@ -348,8 +369,8 @@ class Case(object):
                 "url": str,
             })
 
-        That will check that ``host`` configuration, if given, is valid based on ``validate_host_port`` function, and
+        That will check that ``host`` configuration, if given, is valid based on :func:`validate_host_port`, and will
-        will also check that ``url`` configuration, if given, is a ``str`` instance.
+        also check that ``url`` configuration, if given, is a ``str`` instance.
         """
         self._schema = schema
 
@@ -375,7 +396,7 @@ class Or(object):
 
         The outer :class:`Or` is used to define that ``host`` and ``hosts`` are possible options in this scope.
         The inner :class`Or` in the ``hosts`` key value is used to define that ``hosts`` option is valid if either of
-            the functions ``comma_separated_host_port`` or ``validate_host_port`` succeed to validate it.
+            :func:`comma_separated_host_port` or :func:`validate_host_port` succeed to validate it.
         """
         self.args = args
 
@@ -417,12 +438,12 @@ class Directory(object):
         self.contains_executable = contains_executable
 
     def _check_executables(self, path: OptionalType[str] = None) -> Iterator[Result]:
-        """Check that all executables from contains_executable list exist within the given directory or within PATH.
+        """Check that all executables from contains_executable list exist within the given directory or within ``PATH``.
 
         :param path: optional path to the base directory against which executables will be validated.
-                     If not provided, check within PATH.
+                     If not provided, check within ``PATH``.
-        :rtype: Iterator[:class:`Result`] objects with the error message containing the name of the executable,
+
-                if any check fails.
+        :yields: objects with the error message containing the name of the executable, if any check fails.
         """
         for program in self.contains_executable or []:
             if not shutil.which(program, path=path):
@@ -432,8 +453,9 @@ class Directory(object):
         """Check if the expected paths and executables can be found under *name* directory.
 
         :param name: path to the base directory against which paths and executables will be validated.
-                     Check against PATH if name is not provided.
+                     Check against ``PATH`` if name is not provided.
-        :rtype: Iterator[:class:`Result`] objects with the error message related to the failure, if any check fails.
+
+        :yields: objects with the error message related to the failure, if any check fails.
         """
         if not name:
             yield from self._check_executables()
@@ -481,10 +503,11 @@ class Schema(object):
     be performed against each one of them. The validations will be performed whenever the :class:`Schema` object is
     called, or its :func:`validate` method is called.
 
-    :ivar validator: validator of the configuration schema. Can be any of these
+    :ivar validator: validator of the configuration schema. Can be any of these:
+
         * :class:`str`: defines that a string value is required; or
-        * :class:`type`: any subclass of `type`, defines that a value of the given type is required; or
+        * :class:`type`: any subclass of :class:`type`, defines that a value of the given type is required; or
-        * `callable`: any callable object, defines that validation will follow the code defined in the callable
+        * ``callable``: any callable object, defines that validation will follow the code defined in the callable
           object. If the callable object contains an ``expected_type`` attribute, then it will check if the
           configuration value is of the expected type before calling the code of the callable object; or
         * :class:`list`: list representing one or more values in the configuration; or
@@ -503,9 +526,10 @@ class Schema(object):
             nodes, when it performs checks of the actual setting values.
 
         :param validator: validator of the configuration schema. Can be any of these:
+
             * :class:`str`: defines that a string value is required; or
             * :class:`type`: any subclass of :class:`type`, defines that a value of the given type is required; or
-            * `callable`: Any callable object, defines that validation will follow the code defined in the callable
+            * ``callable``: Any callable object, defines that validation will follow the code defined in the callable
               object. If the callable object contains an ``expected_type`` attribute, then it will check if the
               configuration value is of the expected type before calling the code of the callable object; or
             * :class:`list`: list representing it expects to contain one or more values in the configuration; or
@@ -515,14 +539,18 @@ class Schema(object):
             to stop.
 
             If *validator* is a :class:`dict`, then you should follow these rules:
+
             * For the keys it can be either:
+
                 * A :class:`str` instance. It will be the name of the configuration option; or
                 * An :class:`Optional` instance. The ``name`` attribute of that object will be the name of the
                   configuration option, and that class makes this configuration option as optional to the
                   user, allowing it to not be specified in the YAML; or
                 * An :class:`Or` instance. The ``args`` attribute of that object will contain a tuple of
                     configuration option names. At least one of them should be specified by the user in the YAML;
+
             * For the values it can be either:
+
                 * A new :class:`dict` instance. It will represent a new level in the YAML configuration tree; or
                 * A :class:`Case` instance. This is required if the key of this value is an :class:`Or` instance,
                   and the :class:`Case` instance is used to map each of the ``args`` in :class:`Or` to their
@@ -549,6 +577,7 @@ class Schema(object):
             })
 
         This sample schema defines that your YAML configuration follows these rules:
+
             * It must contain an ``application_name`` entry which value should be a :class:`str` instance;
             * It must contain a ``bind.host`` entry which value should be valid as per function ``validate_host``;
             * It must contain a ``bind.port`` entry which value should be an :class:`int` instance;
@@ -565,6 +594,7 @@ class Schema(object):
         """Perform validation of data using the rules defined in this schema.
 
         :param data: configuration to be validated against ``validator``.
+
         :returns: list of errors identified while validating the *data*, if any.
         """
         errors: List[str] = []
@@ -586,7 +616,8 @@ class Schema(object):
                 :class:`Or`), then it iterates over it to validate each of the corresponding entries in *data* argument.
 
         :param data: configuration to be validated against ``validator``.
-        :rtype: Iterator[:class:`Result`] objects with the error message related to the failure, if any check fails.
+
+        :yields: objects with the error message related to the failure, if any check fails.
         """
         self.data = data
 
@@ -627,7 +658,7 @@ class Schema(object):
 
         Only :class:`dict`, :class:`list`, :class:`Directory` and :class:`Or` objects are considered iterable objects.
 
-        :rtype: Iterator[:class:`Result`] objects with the error message related to the failure, if any check fails.
+        :yields: objects with the error message related to the failure, if any check fails.
         """
         if isinstance(self.validator, dict):
             if not isinstance(self.data, dict):
@@ -655,7 +686,7 @@ class Schema(object):
     def iter_dict(self) -> Iterator[Result]:
         """Iterate over a :class:`dict` based ``validator`` to validate the corresponding entries in ``data``.
 
-        :rtype: Iterator[:class:`Result`] objects with the error message related to the failure, if any check fails.
+        :yields: objects with the error message related to the failure, if any check fails.
         """
         # One key in `validator` attribute (`key` variable) can be mapped to one or more keys in `data` attribute (`d`
         # variable), depending on the `key` type.
@@ -678,12 +709,12 @@ class Schema(object):
                                      path=(d + ("." + v.path if v.path else "")), level=v.level, data=v.data)
 
     def iter_or(self) -> Iterator[Result]:
-        """Perform all validations defined in an `Or` object for a given configuration option.
+        """Perform all validations defined in an :class:`Or` object for a given configuration option.
 
         This method can be only called against leaf nodes in the configuration tree. :class:`Or` objects defined in the
         ``validator`` keys will be handled by :func:`iter_dict` method.
 
-        :rtype: Iterator[:class:`Result`] objects with the error message related to the failure, if any check fails.
+        :yields: objects with the error message related to the failure, if any check fails.
         """
         results: List[Result] = []
         for a in self.validator.args:
@@ -709,7 +740,7 @@ class Schema(object):
 
         :param key: key from the ``validator`` attribute.
 
-        :rtype: Iterator[str], keys that should be used to access corresponding value in the ``data`` attribute.
+        :yields: keys that should be used to access corresponding value in the ``data`` attribute.
         """
         # If the key was defined as a `str` object in `validator` attribute, then it is already the final key to access
         # the `data` dictionary.
@@ -736,12 +767,11 @@ class Schema(object):
 
 
 def _get_type_name(python_type: Any) -> str:
-    """Get a user friendly name for a given Python type.
+    """Get a user-friendly name for a given Python type.
 
     :param python_type: Python type which user friendly name should be taken.
 
-    Returns:
+    :returns: User friendly name of the given Python type.
-        User friendly name of the given Python type.
     """
     types: Dict[Any, str] = {str: 'a string', int: 'an integer', float: 'a number',
                              bool: 'a boolean', list: 'an array', dict: 'a dictionary'}
@@ -762,11 +792,11 @@ def assert_(condition: bool, message: str = "Wrong value") -> None:
 class IntValidator(object):
     """Validate an integer setting.
 
-    :cvar expected_type: the expect Python type for an integer setting (:class:`int`).
+    :cvar expected_type: the expected Python type for an integer setting (:class:`int`).
     :ivar min: minimum allowed value for the setting, if any.
     :ivar max: maximum allowed value for the setting, if any.
-    :ivar base_unit: the base unit to convert the value to before checking if it's within `min` and `max` range.
+    :ivar base_unit: the base unit to convert the value to before checking if it's within *min* and *max* range.
-    :ivar raise_assert: if an ``assert`` call should be performed regarding expected type and valid range.
+    :ivar raise_assert: if an ``assert`` test should be performed regarding expected type and valid range.
     """
 
     expected_type = int
@@ -778,7 +808,7 @@ class IntValidator(object):
         :param min: minimum allowed value for the setting, if any.
         :param max: maximum allowed value for the setting, if any.
         :param base_unit: the base unit to convert the value to before checking if it's within *min* and *max* range.
-        :param raise_assert: if an ``assert`` call should be performed regarding expected type and valid range.
+        :param raise_assert: if an ``assert`` test should be performed regarding expected type and valid range.
         """
         self.min = min
         self.max = max
@@ -789,11 +819,13 @@ class IntValidator(object):
         """Check if *value* is a valid integer and within the expected range.
 
         .. note::
-            If ``raise_assert`` is ``True`` and *value* is not valid, then an ``AssertionError`` will be triggered.
+            If ``raise_assert`` is ``True`` and *value* is not valid, then an :class:`AssertionError` will be triggered.
+
         :param value: value to be checked against the rules defined for this :class:`IntValidator` instance.
+
         :returns: ``True`` if *value* is valid and within the expected range.
         """
-        value = parse_int(value, self.base_unit) or ""
+        value = parse_int(value, self.base_unit)
         ret = isinstance(value, int)\
             and (self.min is None or value >= self.min)\
             and (self.max is None or value <= self.max)

patroni/version.py

@@ -2,4 +2,4 @@
 
 :var __version__: the current Patroni version.
 """
-__version__ = '3.1.0'
+__version__ = '3.1.2'

requirements.docs.txt

@@ -0,0 +1,5 @@
+sphinx>=4
+sphinx_rtd_theme>1
+sphinxcontrib-apidoc
+sphinx-github-style
+pyyaml

tests/__init__.py

@@ -118,11 +118,31 @@ class MockCursor(object):
                                '"state":"streaming","sync_state":"async","sync_priority":0}]'
             now = datetime.datetime.now(tzutc)
             self.results = [(now, 0, '', 0, '', False, now, 'streaming', None, replication_info)]
+        elif sql.startswith('SELECT name, pg_catalog.current_setting(name) FROM pg_catalog.pg_settings'):
+            self.results = [('data_directory', 'data'),
+                            ('hba_file', os.path.join('data', 'pg_hba.conf')),
+                            ('ident_file', os.path.join('data', 'pg_ident.conf')),
+                            ('max_connections', 42),
+                            ('max_locks_per_transaction', 73),
+                            ('max_prepared_transactions', 0),
+                            ('max_replication_slots', 21),
+                            ('max_wal_senders', 37),
+                            ('track_commit_timestamp', 'off'),
+                            ('wal_level', 'replica'),
+                            ('listen_addresses', '6.6.6.6'),
+                            ('port', 1984),
+                            ('archive_command', 'my archive command'),
+                            ('cluster_name', 'my_cluster')]
         elif sql.startswith('SELECT name, setting'):
             self.results = [('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
                             ('wal_block_size', '8192', None, 'integer', 'internal'),
                             ('shared_buffers', '16384', '8kB', 'integer', 'postmaster'),
                             ('wal_buffers', '-1', '8kB', 'integer', 'postmaster'),
+                            ('max_connections', '100', None, 'integer', 'postmaster'),
+                            ('max_prepared_transactions', '0', None, 'integer', 'postmaster'),
+                            ('max_worker_processes', '8', None, 'integer', 'postmaster'),
+                            ('max_locks_per_transaction', '64', None, 'integer', 'postmaster'),
+                            ('max_wal_senders', '5', None, 'integer', 'postmaster'),
                             ('search_path', 'public', None, 'string', 'user'),
                             ('port', '5433', None, 'integer', 'postmaster'),
                             ('listen_addresses', '*', None, 'string', 'postmaster'),
@@ -222,6 +242,7 @@ class PostgresInit(unittest.TestCase):
 
 class BaseTestPostgresql(PostgresInit):
 
+    @patch('time.sleep', Mock())
     def setUp(self):
         super(BaseTestPostgresql, self).setUp()
 

tests/test_bootstrap.py

@@ -238,7 +238,8 @@ class TestBootstrap(BaseTestPostgresql):
         self.p.reload_config({'authentication': {'superuser': {'username': 'p', 'password': 'p'},
                                                  'replication': {'username': 'r', 'password': 'r'},
                                                  'rewind': {'username': 'rw', 'password': 'rw'}},
-                              'listen': '*', 'retry_timeout': 10, 'parameters': {'wal_level': '', 'hba_file': 'foo'}})
+                              'listen': '*', 'retry_timeout': 10,
+                              'parameters': {'wal_level': '', 'hba_file': 'foo', 'max_prepared_transactions': 10}})
         with patch.object(Postgresql, 'major_version', PropertyMock(return_value=110000)), \
                 patch.object(Postgresql, 'restart', Mock()) as mock_restart:
             self.b.post_bootstrap({}, task)

tests/test_config.py

@@ -3,6 +3,7 @@ import sys
 import unittest
 import io
 
+from copy import deepcopy
 from mock import MagicMock, Mock, patch
 from patroni.config import Config, ConfigParseError
 
@@ -22,7 +23,7 @@ class TestConfig(unittest.TestCase):
             self.assertFalse(self.config.set_dynamic_configuration({'foo': 'bar'}))
         self.assertTrue(self.config.set_dynamic_configuration({'standby_cluster': {}, 'postgresql': {
             'parameters': {'cluster_name': 1, 'hot_standby': 1, 'wal_keep_size': 1,
-                           'track_commit_timestamp': 1, 'wal_level': 1}}}))
+                           'track_commit_timestamp': 1, 'wal_level': 1, 'max_connections': '100'}}}))
 
     def test_reload_local_configuration(self):
         os.environ.update({
@@ -149,3 +150,26 @@ class TestConfig(unittest.TestCase):
     @patch('os.path.isdir', Mock(return_value=False))
     def test_invalid_path(self):
         self.assertRaises(ConfigParseError, Config, 'postgres0')
+
+    def test__process_postgresql_parameters(self):
+        expected_params = {
+            'f.oo': 'bar',  # not in ConfigHandler.CMDLINE_OPTIONS
+            'max_connections': 100,  # IntValidator
+            'wal_keep_size': '128MB',  # IntValidator
+            'wal_level': 'hot_standby',  # EnumValidator
+        }
+        input_params = deepcopy(expected_params)
+
+        input_params['max_connections'] = '100'
+        self.assertEqual(self.config._process_postgresql_parameters(input_params), expected_params)
+
+        expected_params['f.oo'] = input_params['f.oo'] = '100'
+        self.assertEqual(self.config._process_postgresql_parameters(input_params), expected_params)
+
+        input_params['wal_level'] = 'cold_standby'
+        expected_params.pop('wal_level')
+        self.assertEqual(self.config._process_postgresql_parameters(input_params), expected_params)
+
+        input_params['max_connections'] = 10
+        expected_params.pop('max_connections')
+        self.assertEqual(self.config._process_postgresql_parameters(input_params), expected_params)

tests/test_ctl.py

@@ -68,6 +68,21 @@ class TestCtl(unittest.TestCase):
 
         self.assertIsNotNone(get_cursor({}, get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, role='any'))
 
+        # Mutually exclusive options
+        with self.assertRaises(PatroniCtlException) as e:
+            get_cursor({}, get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, member_name='other',
+                       role='replica')
+
+        self.assertEqual(str(e.exception), '--role and --member are mutually exclusive options')
+
+        # Invalid member provided
+        self.assertIsNone(get_cursor({}, get_cluster_initialized_with_leader(), None, {'dbname': 'foo'},
+                                     member_name='invalid'))
+
+        # Valid member provided
+        self.assertIsNotNone(get_cursor({}, get_cluster_initialized_with_leader(), None, {'dbname': 'foo'},
+                                        member_name='other'))
+
     def test_parse_dcs(self):
         assert parse_dcs(None) is None
         assert parse_dcs('localhost') == {'etcd': {'host': 'localhost:2379'}}
@@ -231,11 +246,17 @@ class TestCtl(unittest.TestCase):
                 rows = query_member({}, None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
 
         with patch('patroni.ctl.get_cursor', Mock(return_value=None)):
+            # No role nor member given -- generic message
             rows = query_member({}, None, None, None, None, None, 'SELECT pg_catalog.pg_is_in_recovery()', {})
-            self.assertTrue('No connection to' in str(rows))
+            self.assertTrue('No connection is available' in str(rows))
 
-            rows = query_member({}, None, None, None, 'foo', 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
+            # Member given -- message pointing to member
-            self.assertTrue('No connection to' in str(rows))
+            rows = query_member({}, None, None, None, 'foo', None, 'SELECT pg_catalog.pg_is_in_recovery()', {})
+            self.assertTrue('No connection to member foo' in str(rows))
+
+            # Role given -- message pointing to role
+            rows = query_member({}, None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
+            self.assertTrue('No connection to role replica' in str(rows))
 
         with patch('patroni.ctl.get_cursor', Mock(side_effect=OperationalError('bla'))):
             rows = query_member({}, None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
@@ -402,9 +423,19 @@ class TestCtl(unittest.TestCase):
     @patch('patroni.ctl.get_dcs')
     def test_members(self, mock_get_dcs):
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+
         result = self.runner.invoke(ctl, ['list'])
         assert '127.0.0.1' in result.output
         assert result.exit_code == 0
+        assert 'Citus cluster: alpha -' in result.output
+
+        result = self.runner.invoke(ctl, ['list', '--group', '0'])
+        assert 'Citus cluster: alpha (group: 0, 12345678901) -' in result.output
+
+        with patch('patroni.ctl.load_config', Mock(return_value={'scope': 'alpha'})):
+            result = self.runner.invoke(ctl, ['list'])
+            assert 'Cluster: alpha (12345678901) -' in result.output
+
         with patch('patroni.ctl.load_config', Mock(return_value={})):
             self.runner.invoke(ctl, ['list'])
 

tests/test_ha.py

@@ -1298,6 +1298,39 @@ class TestHa(PostgresInit):
         mock_restart.assert_called_once()
         self.ha.dcs.get_cluster.assert_not_called()
 
+    @patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
+    def test_enable_synchronous_mode(self):
+        self.ha.is_synchronous_mode = true
+        self.ha.has_lock = true
+        self.p.name = 'leader'
+        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(), CaseInsensitiveSet()))
+        self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
+        with patch('patroni.ha.logger.info') as mock_logger:
+            self.ha.run_cycle()
+            self.assertEqual(mock_logger.call_args_list[0][0][0], 'Enabled synchronous replication')
+        self.ha.dcs.write_sync_state = Mock(return_value=None)
+        with patch('patroni.ha.logger.warning') as mock_logger:
+            self.ha.run_cycle()
+            self.assertEqual(mock_logger.call_args[0][0], 'Updating sync state failed')
+
+    @patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
+    def test_inconsistent_synchronous_state(self):
+        self.ha.is_synchronous_mode = true
+        self.ha.has_lock = true
+        self.p.name = 'leader'
+        self.ha.cluster = get_cluster_initialized_without_leader(sync=('leader', 'a'))
+        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet('a'), CaseInsensitiveSet()))
+        self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
+        mock_set_sync = self.p.sync_handler.set_synchronous_standby_names = Mock()
+        with patch('patroni.ha.logger.warning') as mock_logger:
+            self.ha.run_cycle()
+            mock_set_sync.assert_called_once()
+            self.assertTrue(mock_logger.call_args_list[0][0][0].startswith('Inconsistent state between '))
+        self.ha.dcs.write_sync_state = Mock(return_value=None)
+        with patch('patroni.ha.logger.warning') as mock_logger:
+            self.ha.run_cycle()
+            self.assertEqual(mock_logger.call_args[0][0], 'Updating sync state failed')
+
     def test_effective_tags(self):
         self.ha._disable_sync = True
         self.assertEqual(self.ha.get_effective_tags(), {'foo': 'bar', 'nosync': True})

tests/test_kubernetes.py

@@ -308,13 +308,15 @@ class TestKubernetesConfigMaps(BaseTestKubernetes):
         mock_patch_namespaced_pod.assert_called()
         self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['isMaster'], 'false')
         self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['tmp_role'], 'replica')
-
+        mock_patch_namespaced_pod.rest_mock()
-        self.k.touch_member({'state': 'running', 'role': 'standby-leader'})
-        mock_patch_namespaced_pod.assert_called()
-        self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['isMaster'], 'false')
-        self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['tmp_role'], 'standby-leader')
 
         self.k._name = 'p-0'
+        self.k.touch_member({'role': 'standby_leader'})
+        mock_patch_namespaced_pod.assert_called()
+        self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['isMaster'], 'false')
+        self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['tmp_role'], 'master')
+        mock_patch_namespaced_pod.rest_mock()
+
         self.k.touch_member({'role': 'primary'})
         mock_patch_namespaced_pod.assert_called()
         self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['isMaster'], 'true')
@@ -434,6 +436,10 @@ class TestKubernetesEndpoints(BaseTestKubernetes):
         mock_logger_exception.assert_called_once()
         self.assertEqual(('create_config_service failed',), mock_logger_exception.call_args[0])
 
+    @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_endpoints', mock_namespaced_kind, create=True)
+    def test_write_leader_optime(self):
+        self.k.write_leader_optime(12345)
+
 
 def mock_watch(*args):
     return urllib3.HTTPResponse()

tests/test_patroni.py

@@ -40,6 +40,7 @@ class MockFrozenImporter(object):
 @patch('time.sleep', Mock())
 @patch('subprocess.call', Mock(return_value=0))
 @patch('patroni.psycopg.connect', psycopg_connect)
+@patch('urllib3.connection.HTTPConnection.connect', Mock(side_effect=Exception))
 @patch.object(ConfigHandler, 'append_pg_hba', Mock())
 @patch.object(ConfigHandler, 'write_postgresql_conf', Mock())
 @patch.object(ConfigHandler, 'write_recovery_conf', Mock())
@@ -63,6 +64,7 @@ class TestPatroni(unittest.TestCase):
             self.assertRaises(SystemExit, _main)
 
     @patch('pkgutil.iter_importers', Mock(return_value=[MockFrozenImporter()]))
+    @patch('urllib3.connection.HTTPConnection.connect', Mock(side_effect=Exception))
     @patch('sys.frozen', Mock(return_value=True), create=True)
     @patch.object(HTTPServer, '__init__', Mock())
     @patch.object(etcd.Client, 'read', etcd_read)
@@ -106,6 +108,7 @@ class TestPatroni(unittest.TestCase):
     @patch('os.getpid')
     @patch('multiprocessing.Process')
     @patch('patroni.__main__.patroni_main', Mock())
+    @patch('sys.argv', ['patroni.py', 'postgres0.yml'])
     def test_patroni_main(self, mock_process, mock_getpid):
         mock_getpid.return_value = 2
         _main()
@@ -231,8 +234,8 @@ class TestPatroni(unittest.TestCase):
         )
         with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=bad_cluster)):
             # If the api of the running node cannot be reached, this implies unique name
-            with patch.object(self.p, 'request', Mock(side_effect=ConnectionError)):
+            with patch('urllib3.connection.HTTPConnection.connect', Mock(side_effect=ConnectionError)):
                 self.assertIsNone(self.p.ensure_unique_name())
             # Only if the api of the running node is reachable do we throw an error
-            with patch.object(self.p, 'request', Mock()):
+            with patch('urllib3.connection.HTTPConnection.connect', Mock()):
                 self.assertRaises(SystemExit, self.p.ensure_unique_name)

tests/test_postgresql.py

@@ -310,6 +310,17 @@ class TestPostgresql(BaseTestPostgresql):
         self.p.config.write_postgresql_conf()
         self.assertEqual(self.p.config.check_recovery_conf(None), (False, False))
         self.assertEqual(self.p.config.check_recovery_conf(None), (False, False))
+
+        # Config files changed, but can't connect to postgres
+        mock_get_pg_settings.side_effect = PostgresConnectionException('')
+        with patch('patroni.postgresql.config.mtime', mock_mtime):
+            self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
+
+        # Config files didn't change, but postgres crashed or in crash recovery
+        with patch.object(MockPostmaster, 'create_time', Mock(return_value=1234568), create=True):
+            self.assertEqual(self.p.config.check_recovery_conf(None), (False, False))
+
+        # Any other exception raised when executing the query
         mock_get_pg_settings.side_effect = Exception
         with patch('patroni.postgresql.config.mtime', mock_mtime):
             self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
@@ -671,7 +682,7 @@ class TestPostgresql(BaseTestPostgresql):
             self.assertIsNone(self.p.wait_for_startup())
 
     def test_get_server_parameters(self):
-        config = {'parameters': {'wal_level': 'hot_standby'}, 'listen': '0'}
+        config = {'parameters': {'wal_level': 'hot_standby', 'max_prepared_transactions': 100}, 'listen': '0'}
         self.p._global_config = GlobalConfig({'synchronous_mode': True})
         self.p.config.get_server_parameters(config)
         self.p._global_config = GlobalConfig({'synchronous_mode': True, 'synchronous_mode_strict': True})
@@ -704,6 +715,7 @@ class TestPostgresql(BaseTestPostgresql):
         self.assertEqual(self.p.get_primary_timeline(), 1)
 
     @patch.object(Postgresql, 'get_postgres_role_from_data_directory', Mock(return_value='replica'))
+    @patch.object(Postgresql, 'is_running', Mock(return_value=False))
     @patch.object(Bootstrap, 'running_custom_bootstrap', PropertyMock(return_value=True))
     @patch.object(Postgresql, 'controldata', Mock(return_value={'max_connections setting': '200',
                                                                 'max_worker_processes setting': '20',
@@ -947,6 +959,7 @@ class TestPostgresql2(BaseTestPostgresql):
     @patch('patroni.postgresql.CallbackExecutor', Mock())
     @patch.object(Postgresql, 'get_major_version', Mock(return_value=140000))
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
+    @patch.object(Postgresql, 'is_leader', Mock(return_value=False))
     def setUp(self):
         super(TestPostgresql2, self).setUp()
 

tests/test_rewind.py

@@ -93,7 +93,7 @@ class TestRewind(BaseTestPostgresql):
 
         with patch.object(Postgresql, 'is_running', Mock(return_value=True)), \
                 patch.object(MockCursor, 'fetchone',
-                             Mock(side_effect=[(0, 0, 1, 1, 0, 0, 0, 0, 0, None, None, None), Exception])):
+                             Mock(side_effect=[Exception, (0, 0, 1, 1, 0, 0, 0, 0, 0, None, None, None)])):
             self.r.rewind_or_reinitialize_needed_and_possible(self.leader)
 
     @patch.object(CancellableSubprocess, 'call', mock_cancellable_call)

tests/test_slots.py

@@ -7,6 +7,7 @@ from mock import Mock, PropertyMock, patch
 from threading import Thread
 
 from patroni import psycopg
+from patroni.config import GlobalConfig
 from patroni.dcs import Cluster, ClusterConfig, Member, SyncState
 from patroni.postgresql import Postgresql
 from patroni.postgresql.misc import fsync_dir
@@ -28,6 +29,7 @@ class TestSlotsHandler(BaseTestPostgresql):
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def setUp(self):
         super(TestSlotsHandler, self).setUp()
+        self.p._global_config = GlobalConfig({})
         self.s = self.p.slots_handler
         self.p.start()
         config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}}}, 1)
@@ -44,6 +46,7 @@ class TestSlotsHandler(BaseTestPostgresql):
             self.s.sync_replication_slots(cluster, False)
         self.p.set_role('standby_leader')
         with patch.object(SlotsHandler, 'drop_replication_slot', Mock(return_value=(True, False))), \
+                patch.object(GlobalConfig, 'is_standby_cluster', PropertyMock(return_value=True)), \
                 patch('patroni.postgresql.slots.logger.debug') as mock_debug:
             self.s.sync_replication_slots(cluster, False)
             mock_debug.assert_called_once()
@@ -67,6 +70,23 @@ class TestSlotsHandler(BaseTestPostgresql):
             with patch.object(Postgresql, 'major_version', PropertyMock(return_value=90618)):
                 self.s.sync_replication_slots(cluster, False)
 
+    def test_cascading_replica_sync_replication_slots(self):
+        """Test sync with a cascading replica so physical slots are present on a replica."""
+        config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}}}, 1)
+        cascading_replica = Member(0, 'test-2', 28, {
+            'state': 'running', 'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
+            'tags': {'replicatefrom': 'postgresql0'}
+        })
+        cluster = Cluster(True, config, self.leader, 0,
+                          [self.me, self.other, self.leadermem, cascading_replica],
+                          None, SyncState.empty(), None, {'ls': 10}, None)
+        self.p.set_role('replica')
+        with patch.object(Postgresql, '_query') as mock_query, \
+                patch.object(Postgresql, 'is_leader', Mock(return_value=False)):
+            mock_query.return_value = [('ls', 'logical', 'b', 'a', 5, 12345, 105)]
+            ret = self.s.sync_replication_slots(cluster, False)
+        self.assertEqual(ret, [])
+
     def test_process_permanent_slots(self):
         config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}},
                                    'ignore_slots': [{'name': 'blabla'}]}, 1)

tests/test_zookeeper.py

@@ -276,6 +276,7 @@ class TestZooKeeper(unittest.TestCase):
         self.assertTrue(self.zk.delete_cluster())
 
     def test_watch(self):
+        self.zk.event.wait = Mock()
         self.zk.watch(None, 0)
         self.zk.event.is_set = Mock(return_value=True)
         self.zk._fetch_status = False

tox.ini

@@ -6,6 +6,7 @@ postgres_matrix =
     pg13: PG_MAJOR = 13
     pg14: PG_MAJOR = 14
     pg15: PG_MAJOR = 15
+    pg16: PG_MAJOR = 16
 psycopg_deps =
     py{37,38,39,310,311}-{lin,win}: psycopg[binary]
     mac: psycopg2-binary
@@ -77,6 +78,7 @@ platform =
     {[common]platforms}
 allowlist_externals =
     rm
+    true
     {env:OPEN_CMD}
 
 [testenv:dep]
@@ -105,7 +107,7 @@ description = Reformat code with black
 deps = black
 commands = black {posargs:patroni tests}
 
-[testenv:pg{12,13,14,15}-docker-build]
+[testenv:pg{12,13,14,15,16}-docker-build]
 description = Build docker containers needed for testing
 labels =
     behave
@@ -123,7 +125,7 @@ commands =
         --file features/Dockerfile
 allowlist_externals = docker
 
-[testenv:pg{12,13,14,15}-docker-behave-{etcd}-{lin,mac}]
+[testenv:pg{12,13,14,15,16}-docker-behave-{etcd}-{lin,mac}]
 description = Run behaviour tests in patroni-dev docker container
 setenv =
     etcd: DCS=etcd
@@ -132,7 +134,7 @@ setenv =
 labels =
     behave
 depends =
-    pg{11,12,13,14,15}-docker-build
+    pg{11,12,13,14,15,16}-docker-build
 
 # There's a bug which affects calling multiple envs on the command line
 # This should be a valid command: tox -e 'py{36,37,38,39,310,311}-behave-{env:DCS}-lin'
@@ -174,24 +176,52 @@ platform =
     {[common]platforms}
 
 [testenv:docs-{lin,mac,win}]
-description = Build Sphinx documentation
+description = Build Sphinx documentation in HTML format
 labels:
     docs
 deps =
-    sphinx>=4
+    -r requirements.docs.txt
-    sphinx_rtd_theme
+    -r requirements.txt
+    psycopg[binary]
+    psycopg2-binary
 commands =
     sphinx-build \
     -d "{envtmpdir}{/}doctree" docs "{toxworkdir}{/}docs_out" \
     --color -b html \
+    -T -E -W --keep-going \
     {posargs}
 commands_post =
     - {tty:{env:OPEN_CMD} "{toxworkdir}{/}docs_out{/}index.html":true:}
 allowlist_externals =
+    true
     {env:OPEN_CMD}
 platform =
     {[common]platforms}
 
+[testenv:pdf-{lin,mac,win}]
+description = Build Sphinx documentation in PDF format
+labels:
+    docs
+deps =
+    -r requirements.docs.txt
+    -r requirements.txt
+    psycopg[binary]
+    psycopg2-binary
+commands =
+    python -m sphinx -T -E -b latex -d _build/doctrees -D language=en . pdf
+    - latexmk -r pdf/latexmkrc -cd -C pdf/Patroni.tex
+    latexmk -r pdf/latexmkrc -cd -pdf -f -dvi- -ps- -jobname=Patroni -interaction=nonstopmode pdf/Patroni.tex
+
+commands_post =
+    - {tty:{env:OPEN_CMD} "pdf{/}Patroni.pdf":true:}
+allowlist_externals =
+    true
+    latexmk
+    {env:OPEN_CMD}
+platform =
+    {[common]platforms}
+change_dir = docs
+
 [flake8]
 max-line-length = 120
 ignore = D401,W503