mirror of
https://github.com/outbackdingo/patroni.git
synced 2026-08-27 16:10:10 +00:00
Compare commits
22
Commits
@@ -45,8 +45,8 @@ def install_packages(what):
|
||||
packages['exhibitor'] = packages['zookeeper']
|
||||
packages = packages.get(what, [])
|
||||
ver = versions.get(what)
|
||||
if float(ver) >= 15:
|
||||
packages += ['postgresql-{0}-citus-11.2'.format(ver)]
|
||||
if float(ver) == 15:
|
||||
packages += ['postgresql-{0}-citus-12.0'.format(ver)]
|
||||
subprocess.call(['sudo', 'apt-get', 'update', '-y'])
|
||||
return subprocess.call(['sudo', 'apt-get', 'install', '-y', 'postgresql-' + ver, 'expect-dev'] + packages)
|
||||
|
||||
|
||||
@@ -1 +1 @@
|
||||
versions = {'etcd': '9.6', 'etcd3': '14', 'consul': '13', 'exhibitor': '12', 'raft': '11', 'kubernetes': '15'}
|
||||
versions = {'etcd': '9.6', 'etcd3': '16', 'consul': '13', 'exhibitor': '12', 'raft': '11', 'kubernetes': '15'}
|
||||
|
||||
@@ -5,6 +5,7 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
- 'REL_[0-9]+_[0-9]+'
|
||||
|
||||
env:
|
||||
CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}
|
||||
@@ -173,7 +174,7 @@ jobs:
|
||||
|
||||
- uses: jakebailey/pyright-action@v1
|
||||
with:
|
||||
version: 1.1.320
|
||||
version: 1.1.326
|
||||
|
||||
docs:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
+1
-6
@@ -33,7 +33,7 @@ nosetests.xml
|
||||
coverage.xml
|
||||
htmlcov
|
||||
junit.xml
|
||||
features/output*
|
||||
features/output
|
||||
dummy
|
||||
|
||||
# Translations
|
||||
@@ -48,12 +48,10 @@ pgpass
|
||||
scm-source.json
|
||||
|
||||
# Sphinx-generated documentation
|
||||
docs/_build/
|
||||
docs/build/
|
||||
docs/source/_static/
|
||||
docs/source/_templates/
|
||||
docs/modules/
|
||||
docs/pdf/
|
||||
|
||||
# Pycharm IDE
|
||||
.idea/
|
||||
@@ -66,6 +64,3 @@ venv*/
|
||||
|
||||
# Default test data directory
|
||||
data/
|
||||
|
||||
# macOS
|
||||
**/.DS_Store
|
||||
|
||||
+5
-4
@@ -12,7 +12,7 @@ Patroni is a template for high availability (HA) PostgreSQL solutions using Pyth
|
||||
|
||||
We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely.
|
||||
|
||||
Currently supported PostgreSQL versions: 9.3 to 15.
|
||||
Currently supported PostgreSQL versions: 9.3 to 16.
|
||||
|
||||
**Note to Citus users**: Starting from 3.0 Patroni nicely integrates with the `Citus <https://github.com/citusdata/citus>`__ database extension to Postgres. Please check the `Citus support page <https://github.com/zalando/patroni/blob/master/docs/citus.rst>`__ in the Patroni documentation for more info about how to use Patroni high availability together with a Citus distributed cluster.
|
||||
|
||||
@@ -74,8 +74,9 @@ There are a few options available:
|
||||
|
||||
::
|
||||
|
||||
sudo apt-get install python3-psycopg2 # install psycopg2 module on Debian/Ubuntu
|
||||
sudo yum install python3-psycopg2 # install psycopg2 on RedHat/Fedora/CentOS
|
||||
sudo apt-get install python-psycopg2 # install python2 psycopg2 module on Debian/Ubuntu
|
||||
sudo apt-get install python3-psycopg2 # install python3 psycopg2 module on Debian/Ubuntu
|
||||
sudo yum install python-psycopg2 # install python2 psycopg2 on RedHat/Fedora/CentOS
|
||||
|
||||
2. Install psycopg2 from the binary package
|
||||
|
||||
@@ -93,7 +94,7 @@ There are a few options available:
|
||||
|
||||
::
|
||||
|
||||
pip install psycopg[binary]>=3.0.0
|
||||
pip install psycopg[binary]
|
||||
|
||||
**General installation for pip**
|
||||
|
||||
|
||||
Vendored
BIN
Binary file not shown.
+81
-1
@@ -25,7 +25,83 @@ We report new releases information :ref:`here <releases>`.
|
||||
Technical Requirements/Installation
|
||||
-----------------------------------
|
||||
|
||||
Go :ref:`here <installation>` for guidance on installing and upgrading Patroni on various platforms.
|
||||
**Pre-requirements for Mac OS**
|
||||
|
||||
To install requirements on a Mac, run the following:
|
||||
|
||||
::
|
||||
|
||||
brew install postgresql etcd haproxy libyaml python
|
||||
|
||||
.. _psycopg2_install_options:
|
||||
|
||||
**Psycopg**
|
||||
|
||||
Starting from `psycopg2-2.8 <http://initd.org/psycopg/articles/2019/04/04/psycopg-28-released/>`__ the binary version of psycopg2 will no longer be installed by default. Installing it from the source code requires C compiler and postgres+python dev packages.
|
||||
Since in the python world it is not possible to specify dependency as ``psycopg2 OR psycopg2-binary`` you will have to decide how to install it.
|
||||
|
||||
There are a few options available:
|
||||
|
||||
1. Use the package manager from your distro
|
||||
|
||||
::
|
||||
|
||||
sudo apt-get install python-psycopg2 # install python2 psycopg2 module on Debian/Ubuntu
|
||||
sudo apt-get install python3-psycopg2 # install python3 psycopg2 module on Debian/Ubuntu
|
||||
sudo yum install python-psycopg2 # install python2 psycopg2 on RedHat/Fedora/CentOS
|
||||
|
||||
2. Install psycopg2 from the binary package
|
||||
|
||||
::
|
||||
|
||||
pip install psycopg2-binary
|
||||
|
||||
3. Install psycopg2 from source
|
||||
|
||||
::
|
||||
|
||||
pip install psycopg2>=2.5.4
|
||||
|
||||
4. Use psycopg 3.0 instead of psycopg2
|
||||
|
||||
::
|
||||
|
||||
pip install psycopg[binary]>=3.0.0
|
||||
|
||||
**General installation for pip**
|
||||
|
||||
Patroni can be installed with pip:
|
||||
|
||||
::
|
||||
|
||||
pip install patroni[dependencies]
|
||||
|
||||
where dependencies can be either empty, or consist of one or more of the following:
|
||||
|
||||
etcd or etcd3
|
||||
`python-etcd` module in order to use Etcd as Distributed Configuration Store (DCS)
|
||||
consul
|
||||
`python-consul` module in order to use Consul as DCS
|
||||
zookeeper
|
||||
`kazoo` module in order to use Zookeeper as DCS
|
||||
exhibitor
|
||||
`kazoo` module in order to use Exhibitor as DCS (same dependencies as for Zookeeper)
|
||||
kubernetes
|
||||
`kubernetes` module in order to use Kubernetes as DCS in Patroni
|
||||
raft
|
||||
`pysyncobj` module in order to use python Raft implementation as DCS
|
||||
aws
|
||||
`boto3` in order to use AWS callbacks
|
||||
|
||||
For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is:
|
||||
|
||||
::
|
||||
|
||||
pip install patroni[etcd,aws]
|
||||
|
||||
Note that external tools to call in the replica creation or custom bootstrap scripts (i.e. WAL-E) should be installed
|
||||
independently of Patroni.
|
||||
|
||||
|
||||
.. _running_configuring:
|
||||
|
||||
@@ -89,6 +165,10 @@ Applications Should Not Use Superusers
|
||||
|
||||
When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
|
||||
|
||||
.. |Build Status| image:: https://travis-ci.org/zalando/patroni.svg?branch=master
|
||||
:target: https://travis-ci.org/zalando/patroni
|
||||
.. |Coverage Status| image:: https://coveralls.io/repos/zalando/patroni/badge.svg?branch=master
|
||||
:target: https://coveralls.io/r/zalando/patroni?branch=master
|
||||
|
||||
Testing Your HA Solution
|
||||
--------------------------------------
|
||||
|
||||
+1
-1
@@ -140,7 +140,7 @@ An example of ``patronictl switchover`` on the worker cluster::
|
||||
| work2-1 | 172.27.0.5 | Sync Standby | running | 1 | 0 |
|
||||
| work2-2 | 172.27.0.7 | Leader | running | 1 | |
|
||||
+---------+------------+--------------+---------+----+-----------+
|
||||
Are you sure you want to perform a switchover in the cluster demo, demoting current primary work2-2? [y/N]: y
|
||||
Are you sure you want to switchover cluster demo, demoting current primary work2-2? [y/N]: y
|
||||
2022-12-22 07:02:40.33003 Successfully switched over to "work2-1"
|
||||
+ Citus cluster: demo (group: 2, 7179854924063375386) ------+
|
||||
| Member | Host | Role | State | TL | Lag in MB |
|
||||
|
||||
+1
-17
@@ -54,13 +54,6 @@ apidoc_output_dir = 'modules'
|
||||
apidoc_excluded_paths = excludes
|
||||
apidoc_separate_modules = True
|
||||
|
||||
# Include autodoc for all members, including private ones and the ones that are missing a docstring.
|
||||
autodoc_default_options = {
|
||||
"members": True,
|
||||
"undoc-members": True,
|
||||
"private-members": True,
|
||||
}
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ['_templates']
|
||||
|
||||
@@ -112,10 +105,10 @@ todo_include_todos = True
|
||||
# a list of builtin themes.
|
||||
#
|
||||
|
||||
html_theme = 'sphinx_rtd_theme'
|
||||
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
|
||||
if not on_rtd: # only import and set the theme if we're building docs locally
|
||||
import sphinx_rtd_theme
|
||||
html_theme = 'sphinx_rtd_theme'
|
||||
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
|
||||
|
||||
# Theme options are theme-specific and customize the look and feel of a theme
|
||||
@@ -287,14 +280,6 @@ def doctree_read(app, doctree):
|
||||
toc_tree_node['entries'].remove(e)
|
||||
|
||||
|
||||
def autodoc_skip(app, what, name, obj, would_skip, options):
|
||||
"""Include autodoc of ``__init__`` methods, which are skipped by default."""
|
||||
if name == "__init__":
|
||||
return False
|
||||
return would_skip
|
||||
|
||||
|
||||
|
||||
# A possibility to have an own stylesheet, to add new rules or override existing ones
|
||||
# For the latter case, the CSS specificity of the rules should be higher than the default ones
|
||||
def setup(app):
|
||||
@@ -307,4 +292,3 @@ def setup(app):
|
||||
app.connect('builder-inited', builder_inited)
|
||||
app.connect('env-get-outdated', env_get_outdated)
|
||||
app.connect('doctree-read', doctree_read)
|
||||
app.connect("autodoc-skip-member", autodoc_skip)
|
||||
|
||||
+1
-2
@@ -10,7 +10,7 @@ Patroni is a template for high availability (HA) PostgreSQL solutions using Pyth
|
||||
|
||||
We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely. There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
|
||||
|
||||
Currently supported PostgreSQL versions: 9.3 to 15.
|
||||
Currently supported PostgreSQL versions: 9.3 to 16.
|
||||
|
||||
**Note to Citus users**: Starting from 3.0 Patroni nicely integrates with the `Citus <https://github.com/citusdata/citus>`__ database extension to Postgres. Please check the :ref:`Citus support page <citus>` in the Patroni documentation for more info about how to use Patroni high availability together with a Citus distributed cluster.
|
||||
|
||||
@@ -22,7 +22,6 @@ Currently supported PostgreSQL versions: 9.3 to 15.
|
||||
:caption: Contents:
|
||||
|
||||
README
|
||||
installation
|
||||
patroni_configuration
|
||||
rest_api
|
||||
replica_bootstrap
|
||||
|
||||
@@ -1,201 +0,0 @@
|
||||
.. _installation:
|
||||
|
||||
Installation
|
||||
============
|
||||
|
||||
Pre-requirements for Mac OS
|
||||
---------------------------
|
||||
|
||||
To install requirements on a Mac, run the following:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
brew install postgresql etcd haproxy libyaml python
|
||||
|
||||
.. _psycopg2_install_options:
|
||||
|
||||
Psycopg
|
||||
-------
|
||||
|
||||
Starting from `psycopg2-2.8`_ the binary version of psycopg2 will no longer be installed by default. Installing it from
|
||||
the source code requires C compiler and postgres+python dev packages. Since in the python world it is not possible to
|
||||
specify dependency as ``psycopg2 OR psycopg2-binary`` you will have to decide how to install it.
|
||||
|
||||
There are a few options available:
|
||||
|
||||
1. Use the package manager from your distro
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
sudo apt-get install python3-psycopg2 # install psycopg2 module on Debian/Ubuntu
|
||||
sudo yum install python3-psycopg2 # install psycopg2 on RedHat/Fedora/CentOS
|
||||
|
||||
2. Install psycopg2 from the binary package
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
pip install psycopg2-binary
|
||||
|
||||
3. Install psycopg2 from source
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
pip install psycopg2>=2.5.4
|
||||
|
||||
4. Use psycopg 3.0 instead of psycopg2
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
pip install psycopg[binary]>=3.0.0
|
||||
|
||||
General installation for pip
|
||||
----------------------------
|
||||
|
||||
Patroni can be installed with pip:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
pip install patroni[dependencies]
|
||||
|
||||
where ``dependencies`` can be either empty, or consist of one or more of the following:
|
||||
|
||||
etcd or etcd3
|
||||
`python-etcd` module in order to use Etcd as Distributed Configuration Store (DCS)
|
||||
consul
|
||||
`python-consul` module in order to use Consul as DCS
|
||||
zookeeper
|
||||
`kazoo` module in order to use Zookeeper as DCS
|
||||
exhibitor
|
||||
`kazoo` module in order to use Exhibitor as DCS (same dependencies as for Zookeeper)
|
||||
kubernetes
|
||||
`kubernetes` module in order to use Kubernetes as DCS in Patroni
|
||||
raft
|
||||
`pysyncobj` module in order to use python Raft implementation as DCS
|
||||
aws
|
||||
`boto3` in order to use AWS callbacks
|
||||
|
||||
For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
pip install patroni[etcd,aws]
|
||||
|
||||
Note that external tools to call in the replica creation or custom bootstrap scripts (i.e. WAL-E) should be installed
|
||||
independently of Patroni.
|
||||
|
||||
.. _package_installation:
|
||||
|
||||
Package installation on Linux
|
||||
-----------------------------
|
||||
|
||||
Patroni packages may be available for your operating system, produced by the Postgres community for:
|
||||
|
||||
* RHEL, RockyLinux, AlmaLinux;
|
||||
* Debian and Ubuntu;
|
||||
* SUSE Enterprise Linux.
|
||||
|
||||
You can also find packages for direct dependencies of Patroni, like python modules that might not be available in
|
||||
the official operating system repositories.
|
||||
|
||||
For more information see the `PGDG repository`_ documentation.
|
||||
|
||||
If you are on a RedHat Enterprise Linux derivative operating system you may also require packages from EPEL, see
|
||||
`EPEL repository`_ documentation.
|
||||
|
||||
Once you have installed the PGDG repository for your OS you can install patroni.
|
||||
|
||||
.. note::
|
||||
|
||||
Patroni packages are not maintained by the Patroni developers, but rather by the Postgres community. If you
|
||||
require support please first try connecting on `Postgres slack`_.
|
||||
|
||||
Installing on Debian derivatives
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
With PGDG repo installed, see :ref:`above <package_installation>`, install Patroni via apt run:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
apt-get install patroni
|
||||
|
||||
Installing on RedHat derivatives
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
With PGDG repo installed, see :ref:`above <package_installation>`, install patroni with an etcd DCS via dnf on RHEL 9
|
||||
(and derivatives) run:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
dnf install patroni patroni-etcd
|
||||
|
||||
You can install etcd from PGDG if your RedHat derivative distribution does not provide packages. On the nodes that will
|
||||
host the DCS run:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
dnf install 'dnf-command(config-manager)'
|
||||
dnf config-manager --enable pgdg-rhel9-extras
|
||||
dnf install etcd
|
||||
|
||||
You can replace the version of RHEL with `8` in the repo to make `pgdg-rhel8-extras` if needed. The repo name is still
|
||||
`pgdg-rhelN-extras` on RockyLinux, AlmaLinux, Oracle Linux, etc...
|
||||
|
||||
Installing on SUSE Enterprise Linux
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
You might need to enable the SUSE PackageHub repositories for some dependencies. see `SUSE PackageHub`_ documentation.
|
||||
|
||||
For SLES 15 with PGDG repo installed, see :ref:`above <package_installation>`, you can install patroni using:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
zypper install patroni patroni-etcd
|
||||
|
||||
With the SUSE PackageHub repo enabled you can also install etcd:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
SUSEConnect -p PackageHub/15.5/x86_64
|
||||
zypper install etcd
|
||||
|
||||
Upgrading
|
||||
---------
|
||||
|
||||
Upgrading patroni is a very simple process, just update the software installation and restart the Patroni daemon on
|
||||
each node in the cluster.
|
||||
|
||||
However, restarting the Patroni daemon will result in a Postgres database restart. In some situations this may cause
|
||||
a failover of the primary node in your cluster, therefore it is recommended to put the cluster into maintenance mode
|
||||
until the Patroni daemon restart has been completed.
|
||||
|
||||
To put the cluster in maintenance mode, run the following command on one of the patroni nodes:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
patronictl pause --wait
|
||||
|
||||
Then on each node in the cluster, perform the package upgrade required for your OS:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
apt-get update && apt-get install patroni patroni-etcd
|
||||
|
||||
Restart the patroni daemon process on each node:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
systemctl restart patroni
|
||||
|
||||
Then finally resume monitoring of Postgres with patroni to take it out of maintenance mode:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
patronictl resume --wait
|
||||
|
||||
The cluster will now be full operational with the new version of Patroni.
|
||||
|
||||
.. _psycopg2-2.8: http://initd.org/psycopg/articles/2019/04/04/psycopg-28-released/
|
||||
.. _PGDG repository: https://www.postgresql.org/download/linux/
|
||||
.. _EPEL repository: https://docs.fedoraproject.org/en-US/epel/
|
||||
.. _SUSE PackageHub: https://packagehub.suse.com/how-to-use/
|
||||
.. _Postgres slack: http://pgtreats.info/slack-invite
|
||||
@@ -44,6 +44,7 @@ Some of the PostgreSQL parameters **must hold the same values on the primary and
|
||||
- **max_worker_processes**: 8
|
||||
- **max_prepared_transactions**: 0
|
||||
- **wal_level**: hot_standby
|
||||
- **wal_log_hints**: on
|
||||
- **track_commit_timestamp**: off
|
||||
|
||||
For the parameters below, PostgreSQL does not require equal values among the primary and all the replicas. However, considering the possibility of a replica to become the primary at any time, it doesn't really make sense to set them differently; therefore, **Patroni restricts setting their values to the** :ref:`dynamic configuration <dynamic_configuration>`.
|
||||
@@ -61,7 +62,6 @@ There are some other Postgres parameters controlled by Patroni:
|
||||
- **port** - is set either from ``postgresql.listen`` or from ``PATRONI_POSTGRESQL_LISTEN`` environment variable
|
||||
- **cluster_name** - is set either from ``scope`` or from ``PATRONI_SCOPE`` environment variable
|
||||
- **hot_standby: on**
|
||||
- **wal_log_hints: on** - for Postgres 9.4 and newer.
|
||||
|
||||
To be on the safe side parameters from the above lists are not written into ``postgresql.conf``, but passed as a list of arguments to the ``pg_ctl start`` which gives them the highest precedence, even above `ALTER SYSTEM <https://www.postgresql.org/docs/current/static/sql-altersystem.html>`__
|
||||
|
||||
@@ -90,42 +90,6 @@ The parameters would be applied in the following order (run-time are given the h
|
||||
This allows configuration for all the nodes (2), configuration for a specific node using ``ALTER SYSTEM`` (3) and ensures that parameters essential to the running of Patroni are enforced (4), as well as leaves room for configuration tools that manage `postgresql.conf` directly without involving Patroni (1).
|
||||
|
||||
|
||||
PostgreSQL parameters that touch shared memory
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
PostgreSQL has some parameters that determine the size of the shared memory used by them:
|
||||
|
||||
- **max_connections**
|
||||
- **max_prepared_transactions**
|
||||
- **max_locks_per_transaction**
|
||||
- **max_wal_senders**
|
||||
- **max_worker_processes**
|
||||
|
||||
Changing these parameters require a PostgreSQL restart to take effect, and their shared memory structures cannot be smaller on the standby nodes than on the primary node.
|
||||
|
||||
As explained before, Patroni restrict changing their values through :ref:`dynamic configuration <dynamic_configuration>`, which usually consists of:
|
||||
|
||||
1. Applying changes through ``patronictl edit-config`` (or via REST API ``/config`` endpoint)
|
||||
2. Restarting nodes through ``patronictl restart`` (or via REST API ``/restart`` endpoint)
|
||||
|
||||
**Note:** please keep in mind that you should perform a restart of the PostgreSQL nodes through ``patronictl restart`` command, or via REST API ``/restart`` endpoint. An attempt to restart PostgreSQL by restarting the Patroni daemon, e.g. by executing ``systemctl restart patroni``, can cause a failover to occur in the cluster, if you are restarting the primary node.
|
||||
|
||||
However, as those settings manage shared memory, some extra care should be taken when restarting the nodes:
|
||||
|
||||
* If you want to **increase** the value of any of those settings:
|
||||
|
||||
1. Restart all standbys first
|
||||
2. Restart the primary after that
|
||||
|
||||
* If you want to **decrease** the value of any of those settings:
|
||||
|
||||
1. Restart the primary first
|
||||
2. Restart all standbys after that
|
||||
|
||||
**Note:** if you attempt to restart all nodes in one go after **decreasing** the value of any of those settings, Patroni will ignore the change and restart the standby with the original setting value, thus requiring that you restart the standbys again later. Patroni does that to prevent the standby to enter in an infinite crash loop, because PostgreSQL quits with a `FATAL` message if you attempt to set any of those parameters to a value lower than what is visible in ``pg_controldata`` on the Standby node. In other words, we can only decrease the setting on the standby once its ``pg_controldata`` is up-to-date with the primary in regards to these changes on the primary.
|
||||
|
||||
More information about that can be found at `PostgreSQL Administrator's Overview <https://www.postgresql.org/docs/current/hot-standby.html#HOT-STANDBY-ADMIN>`__.
|
||||
|
||||
Patroni configuration parameters
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
|
||||
+1
-1
@@ -19,7 +19,7 @@ When Patroni runs in a paused mode, it does not change the state of PostgreSQL,
|
||||
|
||||
- For the Postgres primary with the leader lock Patroni updates the lock. If the node with the leader lock stops being the primary (i.e. is demoted manually), Patroni will release the lock instead of promoting the node back.
|
||||
|
||||
- Manual unscheduled restart, manual unscheduled failover/switchover and reinitialize are allowed. No scheduled action is allowed. Manual switchover is only allowed if the node to switch over to is specified.
|
||||
- Manual unscheduled restart, reinitialize and manual failover are allowed. Manual failover is only allowed if the node to failover to is specified. In the paused mode, manual failover does not require a running primary node.
|
||||
|
||||
- If 'parallel' primaries are detected by Patroni, it emits a warning, but does not demote the primary without the leader lock.
|
||||
|
||||
|
||||
@@ -3,6 +3,64 @@
|
||||
Release notes
|
||||
=============
|
||||
|
||||
Version 3.1.1
|
||||
-------------
|
||||
|
||||
**Bugfixes**
|
||||
|
||||
- Reset failsafe state on promote (ChenChangAo)
|
||||
|
||||
If switchover/failover happened shortly after failsafe mode had been activated, the newly promoted primary was demoting itself after failsafe becomes inactive.
|
||||
|
||||
- Silence useless warnings in ``patronictl`` (Alexander Kukushkin)
|
||||
|
||||
If ``patronictl`` uses the same patroni.yaml file as Patroni and can access ``PGDATA`` directory it might have been showing annoying warnings about incorrect values in the global configuration.
|
||||
|
||||
- Explicitly enable synchronous mode for a corner case (Alexander Kukushkin)
|
||||
|
||||
Synchronous mode effectively was never activated if there are no replicas streaming from the primary.
|
||||
|
||||
- Fixed bug with ``0`` integer values validation (Israel Barth Rubio)
|
||||
|
||||
In most cases, it didn't cause any issues, just warnings.
|
||||
|
||||
- Don't return logical slots for standby cluster (Alexander Kukushkin)
|
||||
|
||||
Patroni can't create logical replication slots in the standby cluster, thus they should be ignored if they are defined in the global configuration.
|
||||
|
||||
- Avoid showing docstring in ``patronictl --help`` output (Israel Barth Rubio)
|
||||
|
||||
The ``click`` module needs to get a special hint for that.
|
||||
|
||||
- Fixed bug with ``kubernetes.standby_leader_label_value`` (Alexander Kukushkin)
|
||||
|
||||
This feature effectively never worked.
|
||||
|
||||
- Returned cluster system identifier to the ``patronictl list`` output (Polina Bungina)
|
||||
|
||||
The problem was introduced while implementing the support for Citus, where we need to hide the identifier because it is different for coordinator and all workers.
|
||||
|
||||
- Override ``write_leader_optime`` method in Kubernetes implementation (Alexander Kukushkin)
|
||||
|
||||
The method is supposed to write shutdown LSN to the leader Endpoint/ConfigMap when there are no healthy replicas available to become the new primary.
|
||||
|
||||
- Don't start stopped postgres in pause (Alexander Kukushkin)
|
||||
|
||||
Due to a race condition, Patroni was falsely assuming that the standby should be restarted because some recovery parameters (``primary_conninfo`` or similar) were changed.
|
||||
|
||||
- Fixed bug in ``patronictl query`` command (Israel Barth Rubio)
|
||||
|
||||
It didn't work when only ``-m`` argument was provided or when none of ``-r`` or ``-m`` were provided.
|
||||
|
||||
- Properly treat integer parameters that are used in the command line to start postgres (Polina Bungina)
|
||||
|
||||
If values are supplied as strings and not casted to integer it was resulting in an incorrect calculation of ``max_prepared_transactions`` based on ``max_connections`` for Citus clusters.
|
||||
|
||||
- Don't rely on ``pg_stat_wal_receiver`` when deciding on ``pg_rewind`` (Alexander Kukushkin)
|
||||
|
||||
It could happen that ``received_tli`` reported by ``pg_stat_wal_recevier`` is ahead of the actual replayed timeline, while the timeline reported by ``DENTIFY_SYSTEM`` via replication connection is always correct.
|
||||
|
||||
|
||||
Version 3.1.0
|
||||
-------------
|
||||
|
||||
|
||||
+61
-300
@@ -92,188 +92,26 @@ Monitoring endpoint
|
||||
|
||||
The ``GET /patroni`` is used by Patroni during the leader race. It also could be used by your monitoring system. The JSON document produced by this endpoint has the same structure as the JSON produced by the health check endpoints.
|
||||
|
||||
**Example:** A healthy cluster
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ curl -s http://localhost:8008/patroni | jq .
|
||||
{
|
||||
"state": "running",
|
||||
"postmaster_start_time": "2023-08-18 11:03:37.966359+00:00",
|
||||
"postmaster_start_time": "2019-09-24 09:22:32.555 CEST",
|
||||
"role": "master",
|
||||
"server_version": 150004,
|
||||
"server_version": 110005,
|
||||
"cluster_unlocked": false,
|
||||
"xlog": {
|
||||
"location": 67395656
|
||||
"location": 25624640
|
||||
},
|
||||
"timeline": 1,
|
||||
"replication": [
|
||||
{
|
||||
"usename": "replicator",
|
||||
"application_name": "patroni2",
|
||||
"client_addr": "10.89.0.6",
|
||||
"state": "streaming",
|
||||
"sync_state": "async",
|
||||
"sync_priority": 0
|
||||
},
|
||||
{
|
||||
"usename": "replicator",
|
||||
"application_name": "patroni3",
|
||||
"client_addr": "10.89.0.2",
|
||||
"state": "streaming",
|
||||
"sync_state": "async",
|
||||
"sync_priority": 0
|
||||
}
|
||||
],
|
||||
"dcs_last_seen": 1692356718,
|
||||
"tags": {
|
||||
"clonefrom": true
|
||||
},
|
||||
"database_system_identifier": "7268616322854375442",
|
||||
"timeline": 3,
|
||||
"database_system_identifier": "6739877027151648096",
|
||||
"patroni": {
|
||||
"version": "3.1.0",
|
||||
"scope": "demo",
|
||||
"name": "patroni1"
|
||||
"version": "1.6.0",
|
||||
"scope": "batman"
|
||||
}
|
||||
}
|
||||
|
||||
**Example:** An unlocked cluster
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ curl -s http://localhost:8008/patroni | jq .
|
||||
{
|
||||
"state": "running",
|
||||
"postmaster_start_time": "2023-08-18 11:09:08.615242+00:00",
|
||||
"role": "replica",
|
||||
"server_version": 150004,
|
||||
"xlog": {
|
||||
"received_location": 67419744,
|
||||
"replayed_location": 67419744,
|
||||
"replayed_timestamp": null,
|
||||
"paused": false
|
||||
},
|
||||
"timeline": 1,
|
||||
"replication": [
|
||||
{
|
||||
"usename": "replicator",
|
||||
"application_name": "patroni2",
|
||||
"client_addr": "10.89.0.6",
|
||||
"state": "streaming",
|
||||
"sync_state": "async",
|
||||
"sync_priority": 0
|
||||
},
|
||||
{
|
||||
"usename": "replicator",
|
||||
"application_name": "patroni3",
|
||||
"client_addr": "10.89.0.2",
|
||||
"state": "streaming",
|
||||
"sync_state": "async",
|
||||
"sync_priority": 0
|
||||
}
|
||||
],
|
||||
"cluster_unlocked": true,
|
||||
"dcs_last_seen": 1692356928,
|
||||
"tags": {
|
||||
"clonefrom": true
|
||||
},
|
||||
"database_system_identifier": "7268616322854375442",
|
||||
"patroni": {
|
||||
"version": "3.1.0",
|
||||
"scope": "demo",
|
||||
"name": "patroni1"
|
||||
}
|
||||
}
|
||||
|
||||
**Example:** An unlocked cluster with :ref:`DCS failsafe mode <dcs_failsafe_mode>` enabled
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ curl -s http://localhost:8008/patroni | jq .
|
||||
{
|
||||
"state": "running",
|
||||
"postmaster_start_time": "2023-08-18 11:09:08.615242+00:00",
|
||||
"role": "replica",
|
||||
"server_version": 150004,
|
||||
"xlog": {
|
||||
"location": 67420024
|
||||
},
|
||||
"timeline": 1,
|
||||
"replication": [
|
||||
{
|
||||
"usename": "replicator",
|
||||
"application_name": "patroni2",
|
||||
"client_addr": "10.89.0.6",
|
||||
"state": "streaming",
|
||||
"sync_state": "async",
|
||||
"sync_priority": 0
|
||||
},
|
||||
{
|
||||
"usename": "replicator",
|
||||
"application_name": "patroni3",
|
||||
"client_addr": "10.89.0.2",
|
||||
"state": "streaming",
|
||||
"sync_state": "async",
|
||||
"sync_priority": 0
|
||||
}
|
||||
],
|
||||
"cluster_unlocked": true,
|
||||
"failsafe_mode_is_active": true,
|
||||
"dcs_last_seen": 1692356928,
|
||||
"tags": {
|
||||
"clonefrom": true
|
||||
},
|
||||
"database_system_identifier": "7268616322854375442",
|
||||
"patroni": {
|
||||
"version": "3.1.0",
|
||||
"scope": "demo",
|
||||
"name": "patroni1"
|
||||
}
|
||||
}
|
||||
|
||||
**Example:** A cluster with the :ref:`pause mode <pause>` enabled
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ curl -s http://localhost:8008/patroni | jq .
|
||||
{
|
||||
"state": "running",
|
||||
"postmaster_start_time": "2023-08-18 11:09:08.615242+00:00",
|
||||
"role": "replica",
|
||||
"server_version": 150004,
|
||||
"xlog": {
|
||||
"location": 67420024
|
||||
},
|
||||
"timeline": 1,
|
||||
"replication": [
|
||||
{
|
||||
"usename": "replicator",
|
||||
"application_name": "patroni2",
|
||||
"client_addr": "10.89.0.6",
|
||||
"state": "streaming",
|
||||
"sync_state": "async",
|
||||
"sync_priority": 0
|
||||
},
|
||||
{
|
||||
"usename": "replicator",
|
||||
"application_name": "patroni3",
|
||||
"client_addr": "10.89.0.2",
|
||||
"state": "streaming",
|
||||
"sync_state": "async",
|
||||
"sync_priority": 0
|
||||
}
|
||||
],
|
||||
"pause": true,
|
||||
"dcs_last_seen": 1692356928,
|
||||
"tags": {
|
||||
"clonefrom": true
|
||||
},
|
||||
"database_system_identifier": "7268616322854375442",
|
||||
"patroni": {
|
||||
"version": "3.1.0",
|
||||
"scope": "demo",
|
||||
"name": "patroni1"
|
||||
}
|
||||
}
|
||||
|
||||
Retrieve the Patroni metrics in Prometheus format through the ``GET /metrics`` endpoint.
|
||||
|
||||
@@ -283,70 +121,64 @@ Retrieve the Patroni metrics in Prometheus format through the ``GET /metrics`` e
|
||||
|
||||
# HELP patroni_version Patroni semver without periods. \
|
||||
# TYPE patroni_version gauge
|
||||
patroni_version{scope="batman",name="patroni1"} 020103
|
||||
patroni_version{scope="batman"} 020103
|
||||
# HELP patroni_postgres_running Value is 1 if Postgres is running, 0 otherwise.
|
||||
# TYPE patroni_postgres_running gauge
|
||||
patroni_postgres_running{scope="batman",name="patroni1"} 1
|
||||
patroni_postgres_running{scope="batman"} 1
|
||||
# HELP patroni_postmaster_start_time Epoch seconds since Postgres started.
|
||||
# TYPE patroni_postmaster_start_time gauge
|
||||
patroni_postmaster_start_time{scope="batman",name="patroni1"} 1657656955.179243
|
||||
patroni_postmaster_start_time{scope="batman"} 1657656955.179243
|
||||
# HELP patroni_master Value is 1 if this node is the leader, 0 otherwise.
|
||||
# TYPE patroni_master gauge
|
||||
patroni_master{scope="batman",name="patroni1"} 1
|
||||
# HELP patroni_primary Value is 1 if this node is the leader, 0 otherwise.
|
||||
# TYPE patroni_primary gauge
|
||||
patroni_primary{scope="batman",name="patroni1"} 1
|
||||
patroni_master{scope="batman"} 1
|
||||
# HELP patroni_xlog_location Current location of the Postgres transaction log, 0 if this node is not the leader.
|
||||
# TYPE patroni_xlog_location counter
|
||||
patroni_xlog_location{scope="batman",name="patroni1"} 22320573386952
|
||||
patroni_xlog_location{scope="batman"} 22320573386952
|
||||
# HELP patroni_standby_leader Value is 1 if this node is the standby_leader, 0 otherwise.
|
||||
# TYPE patroni_standby_leader gauge
|
||||
patroni_standby_leader{scope="batman",name="patroni1"} 0
|
||||
patroni_standby_leader{scope="batman"} 0
|
||||
# HELP patroni_replica Value is 1 if this node is a replica, 0 otherwise.
|
||||
# TYPE patroni_replica gauge
|
||||
patroni_replica{scope="batman",name="patroni1"} 0
|
||||
patroni_replica{scope="batman"} 0
|
||||
# HELP patroni_sync_standby Value is 1 if this node is a sync standby replica, 0 otherwise.
|
||||
# TYPE patroni_sync_standby gauge
|
||||
patroni_sync_standby{scope="batman",name="patroni1"} 0
|
||||
patroni_sync_standby{scope="batman"} 0
|
||||
# HELP patroni_xlog_received_location Current location of the received Postgres transaction log, 0 if this node is not a replica.
|
||||
# TYPE patroni_xlog_received_location counter
|
||||
patroni_xlog_received_location{scope="batman",name="patroni1"} 0
|
||||
patroni_xlog_received_location{scope="batman"} 0
|
||||
# HELP patroni_xlog_replayed_location Current location of the replayed Postgres transaction log, 0 if this node is not a replica.
|
||||
# TYPE patroni_xlog_replayed_location counter
|
||||
patroni_xlog_replayed_location{scope="batman",name="patroni1"} 0
|
||||
patroni_xlog_replayed_location{scope="batman"} 0
|
||||
# HELP patroni_xlog_replayed_timestamp Current timestamp of the replayed Postgres transaction log, 0 if null.
|
||||
# TYPE patroni_xlog_replayed_timestamp gauge
|
||||
patroni_xlog_replayed_timestamp{scope="batman",name="patroni1"} 0
|
||||
patroni_xlog_replayed_timestamp{scope="batman"} 0
|
||||
# HELP patroni_xlog_paused Value is 1 if the Postgres xlog is paused, 0 otherwise.
|
||||
# TYPE patroni_xlog_paused gauge
|
||||
patroni_xlog_paused{scope="batman",name="patroni1"} 0
|
||||
patroni_xlog_paused{scope="batman"} 0
|
||||
# HELP patroni_postgres_streaming Value is 1 if Postgres is streaming, 0 otherwise.
|
||||
# TYPE patroni_postgres_streaming gauge
|
||||
patroni_postgres_streaming{scope="batman",name="patroni1"} 1
|
||||
patroni_postgres_streaming{scope="batman"} 1
|
||||
# HELP patroni_postgres_in_archive_recovery Value is 1 if Postgres is replicating from archive, 0 otherwise.
|
||||
# TYPE patroni_postgres_in_archive_recovery gauge
|
||||
patroni_postgres_in_archive_recovery{scope="batman",name="patroni1"} 0
|
||||
patroni_postgres_in_archive_recovery{scope="batman"} 0
|
||||
# HELP patroni_postgres_server_version Version of Postgres (if running), 0 otherwise.
|
||||
# TYPE patroni_postgres_server_version gauge
|
||||
patroni_postgres_server_version{scope="batman",name="patroni1"} 140004
|
||||
patroni_postgres_server_version {scope="batman"} 140004
|
||||
# HELP patroni_cluster_unlocked Value is 1 if the cluster is unlocked, 0 if locked.
|
||||
# TYPE patroni_cluster_unlocked gauge
|
||||
patroni_cluster_unlocked{scope="batman",name="patroni1"} 0
|
||||
patroni_cluster_unlocked{scope="batman"} 0
|
||||
# HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise.
|
||||
# TYPE patroni_postgres_timeline counter
|
||||
patroni_failsafe_mode_is_active{scope="batman",name="patroni1"} 0
|
||||
# HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise.
|
||||
# TYPE patroni_postgres_timeline counter
|
||||
patroni_postgres_timeline{scope="batman",name="patroni1"} 24
|
||||
patroni_postgres_timeline{scope="batman"} 24
|
||||
# HELP patroni_dcs_last_seen Epoch timestamp when DCS was last contacted successfully by Patroni.
|
||||
# TYPE patroni_dcs_last_seen gauge
|
||||
patroni_dcs_last_seen{scope="batman",name="patroni1"} 1677658321
|
||||
patroni_dcs_last_seen{scope="batman"} 1677658321
|
||||
# HELP patroni_pending_restart Value is 1 if the node needs a restart, 0 otherwise.
|
||||
# TYPE patroni_pending_restart gauge
|
||||
patroni_pending_restart{scope="batman",name="patroni1"} 1
|
||||
patroni_pending_restart{scope="batman"} 1
|
||||
# HELP patroni_is_paused Value is 1 if auto failover is disabled, 0 otherwise.
|
||||
# TYPE patroni_is_paused gauge
|
||||
patroni_is_paused{scope="batman",name="patroni1"} 1
|
||||
patroni_is_paused{scope="batman"} 1
|
||||
|
||||
|
||||
Cluster status endpoints
|
||||
@@ -360,24 +192,24 @@ Cluster status endpoints
|
||||
{
|
||||
"members": [
|
||||
{
|
||||
"name": "patroni1",
|
||||
"name": "postgresql0",
|
||||
"host": "127.0.0.1",
|
||||
"port": 5432,
|
||||
"role": "leader",
|
||||
"state": "running",
|
||||
"api_url": "http://10.89.0.4:8008/patroni",
|
||||
"host": "10.89.0.4",
|
||||
"port": 5432,
|
||||
"api_url": "http://127.0.0.1:8008/patroni",
|
||||
"timeline": 5,
|
||||
"tags": {
|
||||
"clonefrom": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "patroni2",
|
||||
"role": "replica",
|
||||
"state": "streaming",
|
||||
"api_url": "http://10.89.0.6:8008/patroni",
|
||||
"host": "10.89.0.6",
|
||||
"name": "postgresql1",
|
||||
"host": "127.0.0.1",
|
||||
"port": 5433,
|
||||
"role": "replica",
|
||||
"state": "running",
|
||||
"api_url": "http://127.0.0.1:8009/patroni",
|
||||
"timeline": 5,
|
||||
"tags": {
|
||||
"clonefrom": true
|
||||
@@ -385,11 +217,9 @@ Cluster status endpoints
|
||||
"lag": 0
|
||||
}
|
||||
],
|
||||
"scope": "demo",
|
||||
"scheduled_switchover": {
|
||||
"at": "2023-09-24T10:36:00+02:00",
|
||||
"from": "patroni1",
|
||||
"to": "patroni3"
|
||||
"at": "2019-09-24T10:36:00+02:00",
|
||||
"from": "postgresql0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -434,7 +264,7 @@ Config endpoint
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ curl -s http://localhost:8008/config | jq .
|
||||
$ curl -s localhost:8008/config | jq .
|
||||
{
|
||||
"ttl": 30,
|
||||
"loop_wait": 10,
|
||||
@@ -445,6 +275,7 @@ Config endpoint
|
||||
"use_pg_rewind": true,
|
||||
"parameters": {
|
||||
"hot_standby": "on",
|
||||
"wal_log_hints": "on",
|
||||
"wal_level": "hot_standby",
|
||||
"max_wal_senders": 5,
|
||||
"max_replication_slots": 5,
|
||||
@@ -471,6 +302,7 @@ Config endpoint
|
||||
"use_pg_rewind": true,
|
||||
"parameters": {
|
||||
"hot_standby": "on",
|
||||
"wal_log_hints": "on",
|
||||
"wal_level": "hot_standby",
|
||||
"max_wal_senders": 5,
|
||||
"max_replication_slots": 5,
|
||||
@@ -494,9 +326,8 @@ Let's check that the node processed this configuration. First of all it should s
|
||||
"location": 2197818976
|
||||
},
|
||||
"patroni": {
|
||||
"version": "1.0",
|
||||
"scope": "batman",
|
||||
"name": "patroni1"
|
||||
"version": "1.0"
|
||||
},
|
||||
"state": "running",
|
||||
"role": "master",
|
||||
@@ -524,6 +355,7 @@ If you want to remove (reset) some setting just patch it with ``null``:
|
||||
"hot_standby": "on",
|
||||
"unix_socket_directories": ".",
|
||||
"wal_level": "hot_standby",
|
||||
"wal_log_hints": "on",
|
||||
"max_wal_senders": 5,
|
||||
"max_replication_slots": 5
|
||||
}
|
||||
@@ -537,7 +369,7 @@ The above call removes ``postgresql.parameters.max_connections`` from the dynami
|
||||
.. code-block:: bash
|
||||
|
||||
$ curl -s -XPUT -d \
|
||||
'{"maximum_lag_on_failover":1048576,"retry_timeout":10,"postgresql":{"use_slots":true,"use_pg_rewind":true,"parameters":{"hot_standby":"on","wal_level":"hot_standby","unix_socket_directories":".","max_wal_senders":5}},"loop_wait":3,"ttl":20}' \
|
||||
'{"maximum_lag_on_failover":1048576,"retry_timeout":10,"postgresql":{"use_slots":true,"use_pg_rewind":true,"parameters":{"hot_standby":"on","wal_log_hints":"on","wal_level":"hot_standby","unix_socket_directories":".","max_wal_senders":5}},"loop_wait":3,"ttl":20}' \
|
||||
http://localhost:8008/config | jq .
|
||||
{
|
||||
"ttl": 20,
|
||||
@@ -549,6 +381,7 @@ The above call removes ``postgresql.parameters.max_connections`` from the dynami
|
||||
"hot_standby": "on",
|
||||
"unix_socket_directories": ".",
|
||||
"wal_level": "hot_standby",
|
||||
"wal_log_hints": "on",
|
||||
"max_wal_senders": 5
|
||||
},
|
||||
"use_pg_rewind": true
|
||||
@@ -560,111 +393,39 @@ The above call removes ``postgresql.parameters.max_connections`` from the dynami
|
||||
Switchover and failover endpoints
|
||||
---------------------------------
|
||||
|
||||
.. _switchover_api:
|
||||
``POST /switchover`` or ``POST /failover``. These endpoints are very similar to each other. There are a couple of minor differences though:
|
||||
|
||||
Switchover
|
||||
^^^^^^^^^^
|
||||
1. The failover endpoint allows to perform a manual failover when there are no healthy nodes, but at the same time it will not allow you to schedule a switchover.
|
||||
|
||||
``/switchover`` endpoint only works when the cluster is healthy (there is a leader). It also allows to schedule a switchover at a given time.
|
||||
2. The switchover endpoint is the opposite. It works only when the cluster is healthy (there is a leader) and allows to schedule a switchover at a given time.
|
||||
|
||||
When calling ``/switchover`` endpoint a candidate can be specified but is not required, in contrast to ``/failover`` endpoint. If a candidate is not provided, all the eligible nodes of the cluster will participate in the leader race after the leader stepped down.
|
||||
|
||||
In the JSON body of the ``POST`` request you must specify the ``leader`` field. The ``candidate`` and the ``scheduled_at`` fields are optional and can be used to schedule a switchover at a specific time.
|
||||
In the JSON body of the ``POST`` request you must specify at least the ``leader`` or ``candidate`` fields and optionally the ``scheduled_at`` field if you want to schedule a switchover at a specific time.
|
||||
|
||||
Depending on the situation, requests might return different HTTP status codes and bodies. Status code **200** is returned when the switchover or failover successfully completed. If the switchover was successfully scheduled, Patroni will return HTTP status code **202**. In case something went wrong, the error status code (one of **400**, **412**, or **503**) will be returned with some details in the response body.
|
||||
|
||||
``DELETE /switchover`` can be used to delete the currently scheduled switchover.
|
||||
|
||||
**Example:** perform a switchover to any healthy standby
|
||||
Example: perform a failover to the specific node:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ curl -s http://localhost:8008/switchover -XPOST -d '{"leader":"postgresql1"}'
|
||||
Successfully switched over to "postgresql2"
|
||||
$ curl -s http://localhost:8009/failover -XPOST -d '{"candidate":"postgresql1"}'
|
||||
Successfully failed over to "postgresql1"
|
||||
|
||||
|
||||
**Example:** perform a switchover to a specific node
|
||||
Example: schedule a switchover from the leader to any other healthy replica in the cluster at a specific time:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ curl -s http://localhost:8008/switchover -XPOST -d \
|
||||
'{"leader":"postgresql1","candidate":"postgresql2"}'
|
||||
Successfully switched over to "postgresql2"
|
||||
$ curl -s http://localhost:8008/switchover -XPOST -d \
|
||||
'{"leader":"postgresql0","scheduled_at":"2019-09-24T12:00+00"}'
|
||||
Switchover scheduled
|
||||
|
||||
|
||||
**Example:** schedule a switchover from the leader to any other healthy standby in the cluster at a specific time.
|
||||
Depending on the situation the request might finish with a different HTTP status code and body. The status code **200** is returned when the switchover or failover successfully completed. If the switchover was successfully scheduled, Patroni will return HTTP status code **202**. In case something went wrong, the error status code (one of **400**, **412** or **503**) will be returned with some details in the response body. For more information please check the source code of ``patroni/api.py:do_POST_failover()`` method.
|
||||
|
||||
.. code-block:: bash
|
||||
- ``DELETE /switchover``: delete the scheduled switchover
|
||||
|
||||
$ curl -s http://localhost:8008/switchover -XPOST -d \
|
||||
'{"leader":"postgresql0","scheduled_at":"2019-09-24T12:00+00"}'
|
||||
Switchover scheduled
|
||||
|
||||
|
||||
Failover
|
||||
^^^^^^^^
|
||||
|
||||
``/failover`` endpoint can be used to perform a manual failover when there are no healthy nodes (e.g. to an asynchronous standby if all synchronous standbys are not healthy enough to promote). However there is no requirement for a cluster not to have leader - failover can also be run on a healthy cluster.
|
||||
|
||||
In the JSON body of the ``POST`` request you must specify the ``candidate`` field. If the ``leader`` field is specified, a switchover is triggered instead.
|
||||
|
||||
**Example:**
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ curl -s http://localhost:8008/failover -XPOST -d '{"candidate":"postgresql1"}'
|
||||
Successfully failed over to "postgresql1"
|
||||
|
||||
.. warning::
|
||||
:ref:`Be very careful <failover_healthcheck>` when using this endpoint, as this can cause data loss in certain situations. In most cases, :ref:`the switchover endpoint <switchover_api>` satisfies the administrator's needs.
|
||||
|
||||
|
||||
``POST /switchover`` and ``POST /failover`` endpoints are used by ``patronictl switchover`` and ``patronictl failover``, respectively.
|
||||
|
||||
``DELETE /switchover`` is used by ``patronictl flush <cluster-name> switchover``.
|
||||
|
||||
.. list-table:: Failover/Switchover comparison
|
||||
:widths: 25 25 25
|
||||
:header-rows: 1
|
||||
|
||||
* -
|
||||
- Failover
|
||||
- Switchover
|
||||
* - Requires leader specified
|
||||
- no
|
||||
- yes
|
||||
* - Requires candidate specified
|
||||
- yes
|
||||
- no
|
||||
* - Can be run in pause
|
||||
- yes
|
||||
- yes (only to a specific candidate)
|
||||
* - Can be scheduled
|
||||
- no
|
||||
- yes (if not in pause)
|
||||
|
||||
.. _failover_healthcheck:
|
||||
|
||||
Healthy standby
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
There are a couple of checks that a member of a cluster should pass to be able to participate in the leader race during a switchover or to become a leader as a failover/switchover candidate:
|
||||
|
||||
- be reachable via Patroni API;
|
||||
- not have ``nofailover`` tag set to ``true``;
|
||||
- have watchdog fully functional (if required by the configuration);
|
||||
- in case of a switchover in a healthy cluster or an automatic failover, not exceed maximum replication lag (``maximum_lag_on_failover`` :ref:`configuration parameter <dynamic_configuration>`);
|
||||
- in case of a switchover in a healthy cluster or an automatic failover, not have a timeline number smaller than the cluster timeline if ``check_timeline`` :ref:`configuration parameter <dynamic_configuration>` is set to ``true``;
|
||||
- in :ref:`synchronous mode <synchronous_mode>`:
|
||||
|
||||
- In case of a switchover (both with and without a candidate): be listed in the ``/sync`` key members;
|
||||
- For a failover in both healthy and unhealthy clusters, this check is omitted.
|
||||
|
||||
.. warning::
|
||||
In case of a manual failover in a cluster without a leader, a candidate will be allowed to promote even if:
|
||||
- it is not in the ``/sync`` key members when synchronous mode is enabled;
|
||||
- its lag exceeds the maximum replication lag allowed;
|
||||
- it has the timeline number smaller than the last known cluster timeline.
|
||||
The ``POST /switchover`` and ``POST failover`` endpoints are used by ``patronictl switchover`` and ``patronictl failover``, respectively.
|
||||
The ``DELETE /switchover`` is used by ``patronictl flush <cluster-name> switchover``.
|
||||
|
||||
|
||||
Restart endpoint
|
||||
|
||||
@@ -30,15 +30,9 @@ Log
|
||||
|
||||
Bootstrap configuration
|
||||
-----------------------
|
||||
|
||||
.. note::
|
||||
Once Patroni has initialized the cluster for the first time and settings have been stored in the DCS, all future
|
||||
changes to the ``bootstrap.dcs`` section of the YAML configuration will not take any effect! If you want to change
|
||||
them please use either ``patronictl edit-config`` or the Patroni :ref:`REST API <rest_api>`.
|
||||
|
||||
- **bootstrap**:
|
||||
|
||||
- **dcs**: This section will be written into `/<namespace>/<scope>/config` of the given configuration store after initializing the new cluster. The global dynamic configuration for the cluster. You can put any of the parameters described in the :ref:`Dynamic Configuration settings <dynamic_configuration>` under ``bootstrap.dcs`` and after Patroni has initialized (bootstrapped) the new cluster, it will write this section into `/<namespace>/<scope>/config` of the configuration store.
|
||||
- **dcs**: This section will be written into `/<namespace>/<scope>/config` of the given configuration store after initializing of new cluster. The global dynamic configuration for the cluster. Under the ``bootstrap.dcs`` you can put any of the parameters described in the :ref:`Dynamic Configuration settings <dynamic_configuration>` and after Patroni initialized (bootstrapped) the new cluster, it will write this section into `/<namespace>/<scope>/config` of the configuration store. All later changes of ``bootstrap.dcs`` will not take any effect! If you want to change them please use either ``patronictl edit-config`` or Patroni :ref:`REST API <rest_api>`.
|
||||
- **method**: custom script to use for bootstrapping this cluster.
|
||||
|
||||
See :ref:`custom bootstrap methods documentation <custom_bootstrap>` for details.
|
||||
|
||||
+2
-5
@@ -27,8 +27,8 @@ RUN set -ex \
|
||||
&& apt-get update \
|
||||
&& apt-get reinstall init-system-helpers \
|
||||
&& apt-get install -y \
|
||||
python3-pip \
|
||||
python3-dev \
|
||||
python3-venv \
|
||||
rsync \
|
||||
curl \
|
||||
gcc \
|
||||
@@ -40,9 +40,7 @@ RUN set -ex \
|
||||
net-tools \
|
||||
iputils-ping \
|
||||
&& rm -rf /var/cache/apt \
|
||||
\
|
||||
&& python3 -m venv /tox \
|
||||
&& /tox/bin/pip install --no-cache-dir tox>=4 \
|
||||
&& python3 -m pip install --no-cache-dir tox \
|
||||
\
|
||||
&& mkdir -p "$PGHOME" \
|
||||
&& sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \
|
||||
@@ -52,7 +50,6 @@ RUN set -ex \
|
||||
&& curl -sL "$ETCDURL/etcd-v$ETCDVERSION-linux-$(dpkg --print-architecture).tar.gz" \
|
||||
| tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl
|
||||
|
||||
ENV PATH="/tox/bin:$PATH"
|
||||
|
||||
# This Dockerfile syntax only works with docker buildx and the syntax
|
||||
# line at the top of this file.
|
||||
|
||||
+22
-144
@@ -1,8 +1,3 @@
|
||||
"""Patroni main entry point.
|
||||
|
||||
Implement ``patroni`` main daemon and expose its entry point.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
@@ -13,7 +8,6 @@ from argparse import Namespace
|
||||
from typing import Any, Dict, Optional, TYPE_CHECKING
|
||||
|
||||
from patroni.daemon import AbstractPatroniDaemon, abstract_main, get_base_arg_parser
|
||||
from patroni.tags import Tags
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from .config import Config
|
||||
@@ -21,33 +15,9 @@ if TYPE_CHECKING: # pragma: no cover
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Patroni(AbstractPatroniDaemon, Tags):
|
||||
"""Implement ``patroni`` command daemon.
|
||||
|
||||
:ivar version: Patroni version.
|
||||
:ivar dcs: DCS object.
|
||||
:ivar watchdog: watchdog handler, if configured to use watchdog.
|
||||
:ivar postgresql: managed Postgres instance.
|
||||
:ivar api: REST API server instance of this node.
|
||||
:ivar request: wrapper for performing HTTP requests.
|
||||
:ivar ha: HA handler.
|
||||
:ivar next_run: time when to run the next HA loop cycle.
|
||||
:ivar scheduled_restart: when a restart has been scheduled to occur, if any. In that case, should contain two keys:
|
||||
* ``schedule``: timestamp when restart should occur;
|
||||
* ``postmaster_start_time``: timestamp when Postgres was last started.
|
||||
"""
|
||||
class Patroni(AbstractPatroniDaemon):
|
||||
|
||||
def __init__(self, config: 'Config') -> None:
|
||||
"""Create a :class:`Patroni` instance with the given *config*.
|
||||
|
||||
Get a connection to the DCS, configure watchdog (if required), set up Patroni interface with Postgres, configure
|
||||
the HA loop and bring the REST API up.
|
||||
|
||||
.. note::
|
||||
Expected to be instantiated and run through :func:`~patroni.daemon.abstract_main`.
|
||||
|
||||
:param config: Patroni configuration.
|
||||
"""
|
||||
from patroni.api import RestApiServer
|
||||
from patroni.dcs import get_dcs
|
||||
from patroni.ha import Ha
|
||||
@@ -71,22 +41,11 @@ class Patroni(AbstractPatroniDaemon, Tags):
|
||||
self.api = RestApiServer(self, self.config['restapi'])
|
||||
self.ha = Ha(self)
|
||||
|
||||
self._tags = self._get_tags()
|
||||
self.tags = self.get_tags()
|
||||
self.next_run = time.time()
|
||||
self.scheduled_restart: Dict[str, Any] = {}
|
||||
|
||||
def load_dynamic_configuration(self) -> None:
|
||||
"""Load Patroni dynamic configuration.
|
||||
|
||||
Load dynamic configuration from the DCS, if `/config` key is available in the DCS, otherwise fall back to
|
||||
``bootstrap.dcs`` section from the configuration file.
|
||||
|
||||
If the DCS connection fails returning the exception :class:`~patroni.exceptions.DCSError` an attempt will be
|
||||
remade every 5 seconds.
|
||||
|
||||
.. note::
|
||||
This method is called only once, at the time when Patroni is started.
|
||||
"""
|
||||
from patroni.exceptions import DCSError
|
||||
while True:
|
||||
try:
|
||||
@@ -121,31 +80,23 @@ class Patroni(AbstractPatroniDaemon, Tags):
|
||||
except Exception:
|
||||
return
|
||||
|
||||
def _get_tags(self) -> Dict[str, Any]:
|
||||
"""Get tags configured for this node, if any.
|
||||
def get_tags(self) -> Dict[str, Any]:
|
||||
return {tag: value for tag, value in self.config.get('tags', {}).items()
|
||||
if tag not in ('clonefrom', 'nofailover', 'noloadbalance', 'nosync') or value}
|
||||
|
||||
:returns: a dictionary of tags set for this node.
|
||||
"""
|
||||
return self._filter_tags(self.config.get('tags', {}))
|
||||
@property
|
||||
def nofailover(self) -> bool:
|
||||
return bool(self.tags.get('nofailover', False))
|
||||
|
||||
@property
|
||||
def nosync(self) -> bool:
|
||||
return bool(self.tags.get('nosync', False))
|
||||
|
||||
def reload_config(self, sighup: bool = False, local: Optional[bool] = False) -> None:
|
||||
"""Apply new configuration values for ``patroni`` daemon.
|
||||
|
||||
Reload:
|
||||
* Cached tags;
|
||||
* Request wrapper configuration;
|
||||
* REST API configuration;
|
||||
* Watchdog configuration;
|
||||
* Postgres configuration;
|
||||
* DCS configuration.
|
||||
|
||||
:param sighup: if it is related to a SIGHUP signal.
|
||||
:param local: if there has been changes to the local configuration file.
|
||||
"""
|
||||
try:
|
||||
super(Patroni, self).reload_config(sighup, local)
|
||||
if local:
|
||||
self._tags = self._get_tags()
|
||||
self.tags = self.get_tags()
|
||||
self.request.reload_config(self.config)
|
||||
if local or sighup and self.api.reload_local_certificate():
|
||||
self.api.reload_config(self.config['restapi'])
|
||||
@@ -156,16 +107,14 @@ class Patroni(AbstractPatroniDaemon, Tags):
|
||||
logger.exception('Failed to reload config_file=%s', self.config.config_file)
|
||||
|
||||
@property
|
||||
def tags(self) -> Dict[str, Any]:
|
||||
"""Tags configured for this node, if any."""
|
||||
return self._tags
|
||||
def replicatefrom(self):
|
||||
return self.tags.get('replicatefrom')
|
||||
|
||||
@property
|
||||
def noloadbalance(self):
|
||||
return bool(self.tags.get('noloadbalance', False))
|
||||
|
||||
def schedule_next_run(self) -> None:
|
||||
"""Schedule the next run of the ``patroni`` daemon main loop.
|
||||
|
||||
Next run is scheduled based on previous run plus value of ``loop_wait`` configuration from DCS. If that has
|
||||
already been exceeded, run the next cycle immediately.
|
||||
"""
|
||||
self.next_run += self.dcs.loop_wait
|
||||
current_time = time.time()
|
||||
nap_time = self.next_run - current_time
|
||||
@@ -179,21 +128,11 @@ class Patroni(AbstractPatroniDaemon, Tags):
|
||||
self.next_run = time.time()
|
||||
|
||||
def run(self) -> None:
|
||||
"""Run ``patroni`` daemon process main loop.
|
||||
|
||||
Start the REST API and keep running HA cycles every ``loop_wait`` seconds.
|
||||
"""
|
||||
self.api.start()
|
||||
self.next_run = time.time()
|
||||
super(Patroni, self).run()
|
||||
|
||||
def _run_cycle(self) -> None:
|
||||
"""Run a cycle of the ``patroni`` daemon main loop.
|
||||
|
||||
Run an HA cycle and schedule the next cycle run. If any dynamic configuration change request is detected, apply
|
||||
the change and cache the new dynamic configuration values in ``patroni.dynamic.json`` file under Postgres data
|
||||
directory.
|
||||
"""
|
||||
logger.info(self.ha.run_cycle())
|
||||
|
||||
if self.dcs.cluster and self.dcs.cluster.config and self.dcs.cluster.config.data \
|
||||
@@ -206,10 +145,6 @@ class Patroni(AbstractPatroniDaemon, Tags):
|
||||
self.schedule_next_run()
|
||||
|
||||
def _shutdown(self) -> None:
|
||||
"""Perform shutdown of ``patroni`` daemon process.
|
||||
|
||||
Shut down the REST API and the HA handler.
|
||||
"""
|
||||
try:
|
||||
self.api.shutdown()
|
||||
except Exception:
|
||||
@@ -221,54 +156,18 @@ class Patroni(AbstractPatroniDaemon, Tags):
|
||||
|
||||
|
||||
def patroni_main(configfile: str) -> None:
|
||||
"""Configure and start ``patroni`` main daemon process.
|
||||
|
||||
:param configfile: path to Patroni configuration file.
|
||||
"""
|
||||
from multiprocessing import freeze_support
|
||||
|
||||
# Windows executables created by PyInstaller are frozen, thus we need to enable frozen support for
|
||||
# :mod:`multiprocessing` to avoid :class:`RuntimeError` exceptions.
|
||||
freeze_support()
|
||||
abstract_main(Patroni, configfile)
|
||||
|
||||
|
||||
def process_arguments() -> Namespace:
|
||||
"""Process command-line arguments.
|
||||
|
||||
Create a basic command-line parser through :func:`~patroni.daemon.get_base_arg_parser`, extend its capabilities by
|
||||
adding these flags and parse command-line arguments.:
|
||||
|
||||
* ``--validate-config`` -- used to validate the Patroni configuration file
|
||||
* ``--generate-config`` -- used to generate Patroni configuration from a running PostgreSQL instance
|
||||
* ``--generate-sample-config`` -- used to generate a sample Patroni configuration
|
||||
|
||||
.. note::
|
||||
If running with ``--generate-config``, ``--generate-sample-config`` or ``--validate-flag`` will exit
|
||||
after generating or validating configuration.
|
||||
|
||||
:returns: parsed arguments, if not running with ``--validate-config`` flag.
|
||||
"""
|
||||
from patroni.config_generator import generate_config
|
||||
|
||||
parser = get_base_arg_parser()
|
||||
group = parser.add_mutually_exclusive_group()
|
||||
group.add_argument('--validate-config', action='store_true', help='Run config validator and exit')
|
||||
group.add_argument('--generate-sample-config', action='store_true',
|
||||
help='Generate a sample Patroni yaml configuration file')
|
||||
group.add_argument('--generate-config', action='store_true',
|
||||
help='Generate a Patroni yaml configuration file for a running instance')
|
||||
parser.add_argument('--dsn', help='Optional DSN string of the instance to be used as a source \
|
||||
for config generation. Superuser connection is required.')
|
||||
parser.add_argument('--validate-config', action='store_true', help='Run config validator and exit')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.generate_sample_config:
|
||||
generate_config(args.configfile, True, None)
|
||||
sys.exit(0)
|
||||
elif args.generate_config:
|
||||
generate_config(args.configfile, False, args.dsn)
|
||||
sys.exit(0)
|
||||
elif args.validate_config:
|
||||
if args.validate_config:
|
||||
from patroni.validator import schema
|
||||
from patroni.config import Config, ConfigParseError
|
||||
|
||||
@@ -282,16 +181,6 @@ def process_arguments() -> Namespace:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Main entrypoint of :mod:`patroni.__main__`.
|
||||
|
||||
Process command-line arguments, ensure :mod:`psycopg2` (or :mod:`psycopg`) attendee the pre-requisites and start
|
||||
``patroni`` daemon process.
|
||||
|
||||
.. note::
|
||||
If running through a Docker container, make the main process take care of init process duties and run
|
||||
``patroni`` daemon as another process. In that case relevant signals received by the main process and forwarded
|
||||
to ``patroni`` daemon process.
|
||||
"""
|
||||
from patroni import check_psycopg
|
||||
|
||||
args = process_arguments()
|
||||
@@ -307,13 +196,7 @@ def main() -> None:
|
||||
|
||||
# Looks like we are in a docker, so we will act like init
|
||||
def sigchld_handler(signo: int, stack_frame: Optional[FrameType]) -> None:
|
||||
"""Handle ``SIGCHLD`` received by main process from ``patroni`` daemon when the daemon terminates.
|
||||
|
||||
:param signo: signal number.
|
||||
:param stack_frame: current stack frame.
|
||||
"""
|
||||
try:
|
||||
# log exit code of all children processes, and break loop when there is none left
|
||||
while True:
|
||||
ret = os.waitpid(-1, os.WNOHANG)
|
||||
if ret == (0, 0):
|
||||
@@ -323,12 +206,7 @@ def main() -> None:
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def passtochild(signo: int, stack_frame: Optional[FrameType]) -> None:
|
||||
"""Forward a signal *signo* from main process to child process.
|
||||
|
||||
:param signo: signal number.
|
||||
:param stack_frame: current stack frame.
|
||||
"""
|
||||
def passtochild(signo: int, stack_frame: Optional[FrameType]):
|
||||
if pid:
|
||||
os.kill(pid, signo)
|
||||
|
||||
|
||||
+148
-77
@@ -12,6 +12,7 @@ import json
|
||||
import logging
|
||||
import time
|
||||
import traceback
|
||||
import dateutil.parser
|
||||
import datetime
|
||||
import os
|
||||
import socket
|
||||
@@ -27,11 +28,11 @@ from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, TYPE_CH
|
||||
|
||||
from . import psycopg
|
||||
from .__main__ import Patroni
|
||||
from .dcs import Cluster
|
||||
from .exceptions import PostgresConnectionException, PostgresException
|
||||
from .manual_failover import ManualFailover
|
||||
from .postgresql.misc import postgres_version_to_int
|
||||
from .utils import deep_compare, enable_keepalive, parse_bool, patch_config, Retry, \
|
||||
RetryFailedError, parse_int, parse_schedule, split_host_port, tzutc, uri, cluster_as_json
|
||||
RetryFailedError, parse_int, split_host_port, tzutc, uri, cluster_as_json
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -197,11 +198,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
response['database_system_identifier'] = patroni.postgresql.sysid
|
||||
if patroni.postgresql.pending_restart:
|
||||
response['pending_restart'] = True
|
||||
response['patroni'] = {
|
||||
'version': patroni.version,
|
||||
'scope': patroni.postgresql.scope,
|
||||
'name': patroni.postgresql.name
|
||||
}
|
||||
response['patroni'] = {'version': patroni.version, 'scope': patroni.postgresql.scope}
|
||||
if patroni.scheduled_restart:
|
||||
response['scheduled_restart'] = patroni.scheduled_restart.copy()
|
||||
del response['scheduled_restart']['postmaster_start_time']
|
||||
@@ -452,10 +449,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
"""
|
||||
cluster = self.server.patroni.dcs.get_cluster(True)
|
||||
global_config = self.server.patroni.config.get_global_config(cluster)
|
||||
|
||||
response = cluster_as_json(cluster, global_config)
|
||||
response['scope'] = self.server.patroni.postgresql.scope
|
||||
self._write_json_response(200, response)
|
||||
self._write_json_response(200, cluster_as_json(cluster, global_config))
|
||||
|
||||
def do_GET_history(self) -> None:
|
||||
"""Handle a ``GET`` request to ``/history`` path.
|
||||
@@ -532,113 +526,113 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
|
||||
metrics: List[str] = []
|
||||
|
||||
labels = f'{{scope="{patroni.postgresql.scope}",name="{patroni.postgresql.name}"}}'
|
||||
scope_label = '{{scope="{0}"}}'.format(patroni.postgresql.scope)
|
||||
metrics.append("# HELP patroni_version Patroni semver without periods.")
|
||||
metrics.append("# TYPE patroni_version gauge")
|
||||
padded_semver = ''.join([x.zfill(2) for x in patroni.version.split('.')]) # 2.0.2 => 020002
|
||||
metrics.append("patroni_version{0} {1}".format(labels, padded_semver))
|
||||
metrics.append("patroni_version{0} {1}".format(scope_label, padded_semver))
|
||||
|
||||
metrics.append("# HELP patroni_postgres_running Value is 1 if Postgres is running, 0 otherwise.")
|
||||
metrics.append("# TYPE patroni_postgres_running gauge")
|
||||
metrics.append("patroni_postgres_running{0} {1}".format(labels, int(postgres['state'] == 'running')))
|
||||
metrics.append("patroni_postgres_running{0} {1}".format(scope_label, int(postgres['state'] == 'running')))
|
||||
|
||||
metrics.append("# HELP patroni_postmaster_start_time Epoch seconds since Postgres started.")
|
||||
metrics.append("# TYPE patroni_postmaster_start_time gauge")
|
||||
postmaster_start_time = postgres.get('postmaster_start_time')
|
||||
postmaster_start_time = (postmaster_start_time - epoch).total_seconds() if postmaster_start_time else 0
|
||||
metrics.append("patroni_postmaster_start_time{0} {1}".format(labels, postmaster_start_time))
|
||||
metrics.append("patroni_postmaster_start_time{0} {1}".format(scope_label, postmaster_start_time))
|
||||
|
||||
metrics.append("# HELP patroni_master Value is 1 if this node is the leader, 0 otherwise.")
|
||||
metrics.append("# TYPE patroni_master gauge")
|
||||
metrics.append("patroni_master{0} {1}".format(labels, int(postgres['role'] in ('master', 'primary'))))
|
||||
metrics.append("patroni_master{0} {1}".format(scope_label, int(postgres['role'] in ('master', 'primary'))))
|
||||
|
||||
metrics.append("# HELP patroni_primary Value is 1 if this node is the leader, 0 otherwise.")
|
||||
metrics.append("# TYPE patroni_primary gauge")
|
||||
metrics.append("patroni_primary{0} {1}".format(labels, int(postgres['role'] in ('master', 'primary'))))
|
||||
metrics.append("patroni_primary{0} {1}".format(scope_label, int(postgres['role'] in ('master', 'primary'))))
|
||||
|
||||
metrics.append("# HELP patroni_xlog_location Current location of the Postgres"
|
||||
" transaction log, 0 if this node is not the leader.")
|
||||
metrics.append("# TYPE patroni_xlog_location counter")
|
||||
metrics.append("patroni_xlog_location{0} {1}".format(labels, postgres.get('xlog', {}).get('location', 0)))
|
||||
metrics.append("patroni_xlog_location{0} {1}".format(scope_label, postgres.get('xlog', {}).get('location', 0)))
|
||||
|
||||
metrics.append("# HELP patroni_standby_leader Value is 1 if this node is the standby_leader, 0 otherwise.")
|
||||
metrics.append("# TYPE patroni_standby_leader gauge")
|
||||
metrics.append("patroni_standby_leader{0} {1}".format(labels, int(postgres['role'] == 'standby_leader')))
|
||||
metrics.append("patroni_standby_leader{0} {1}".format(scope_label, int(postgres['role'] == 'standby_leader')))
|
||||
|
||||
metrics.append("# HELP patroni_replica Value is 1 if this node is a replica, 0 otherwise.")
|
||||
metrics.append("# TYPE patroni_replica gauge")
|
||||
metrics.append("patroni_replica{0} {1}".format(labels, int(postgres['role'] == 'replica')))
|
||||
metrics.append("patroni_replica{0} {1}".format(scope_label, int(postgres['role'] == 'replica')))
|
||||
|
||||
metrics.append("# HELP patroni_sync_standby Value is 1 if this node is a sync standby replica, 0 otherwise.")
|
||||
metrics.append("# TYPE patroni_sync_standby gauge")
|
||||
metrics.append("patroni_sync_standby{0} {1}".format(labels, int(postgres.get('sync_standby', False))))
|
||||
metrics.append("patroni_sync_standby{0} {1}".format(scope_label, int(postgres.get('sync_standby', False))))
|
||||
|
||||
metrics.append("# HELP patroni_xlog_received_location Current location of the received"
|
||||
" Postgres transaction log, 0 if this node is not a replica.")
|
||||
metrics.append("# TYPE patroni_xlog_received_location counter")
|
||||
metrics.append("patroni_xlog_received_location{0} {1}"
|
||||
.format(labels, postgres.get('xlog', {}).get('received_location', 0)))
|
||||
.format(scope_label, postgres.get('xlog', {}).get('received_location', 0)))
|
||||
|
||||
metrics.append("# HELP patroni_xlog_replayed_location Current location of the replayed"
|
||||
" Postgres transaction log, 0 if this node is not a replica.")
|
||||
metrics.append("# TYPE patroni_xlog_replayed_location counter")
|
||||
metrics.append("patroni_xlog_replayed_location{0} {1}"
|
||||
.format(labels, postgres.get('xlog', {}).get('replayed_location', 0)))
|
||||
.format(scope_label, postgres.get('xlog', {}).get('replayed_location', 0)))
|
||||
|
||||
metrics.append("# HELP patroni_xlog_replayed_timestamp Current timestamp of the replayed"
|
||||
" Postgres transaction log, 0 if null.")
|
||||
metrics.append("# TYPE patroni_xlog_replayed_timestamp gauge")
|
||||
replayed_timestamp = postgres.get('xlog', {}).get('replayed_timestamp')
|
||||
replayed_timestamp = (replayed_timestamp - epoch).total_seconds() if replayed_timestamp else 0
|
||||
metrics.append("patroni_xlog_replayed_timestamp{0} {1}".format(labels, replayed_timestamp))
|
||||
metrics.append("patroni_xlog_replayed_timestamp{0} {1}".format(scope_label, replayed_timestamp))
|
||||
|
||||
metrics.append("# HELP patroni_xlog_paused Value is 1 if the Postgres xlog is paused, 0 otherwise.")
|
||||
metrics.append("# TYPE patroni_xlog_paused gauge")
|
||||
metrics.append("patroni_xlog_paused{0} {1}"
|
||||
.format(labels, int(postgres.get('xlog', {}).get('paused', False) is True)))
|
||||
.format(scope_label, int(postgres.get('xlog', {}).get('paused', False) is True)))
|
||||
|
||||
if postgres.get('server_version', 0) >= 90600:
|
||||
metrics.append("# HELP patroni_postgres_streaming Value is 1 if Postgres is streaming, 0 otherwise.")
|
||||
metrics.append("# TYPE patroni_postgres_streaming gauge")
|
||||
metrics.append("patroni_postgres_streaming{0} {1}"
|
||||
.format(labels, int(postgres.get('replication_state') == 'streaming')))
|
||||
.format(scope_label, int(postgres.get('replication_state') == 'streaming')))
|
||||
|
||||
metrics.append("# HELP patroni_postgres_in_archive_recovery Value is 1"
|
||||
" if Postgres is replicating from archive, 0 otherwise.")
|
||||
metrics.append("# TYPE patroni_postgres_in_archive_recovery gauge")
|
||||
metrics.append("patroni_postgres_in_archive_recovery{0} {1}"
|
||||
.format(labels, int(postgres.get('replication_state') == 'in archive recovery')))
|
||||
.format(scope_label, int(postgres.get('replication_state') == 'in archive recovery')))
|
||||
|
||||
metrics.append("# HELP patroni_postgres_server_version Version of Postgres (if running), 0 otherwise.")
|
||||
metrics.append("# TYPE patroni_postgres_server_version gauge")
|
||||
metrics.append("patroni_postgres_server_version {0} {1}".format(labels, postgres.get('server_version', 0)))
|
||||
metrics.append("patroni_postgres_server_version {0} {1}".format(scope_label, postgres.get('server_version', 0)))
|
||||
|
||||
metrics.append("# HELP patroni_cluster_unlocked Value is 1 if the cluster is unlocked, 0 if locked.")
|
||||
metrics.append("# TYPE patroni_cluster_unlocked gauge")
|
||||
metrics.append("patroni_cluster_unlocked{0} {1}".format(labels, int(postgres.get('cluster_unlocked', 0))))
|
||||
metrics.append("patroni_cluster_unlocked{0} {1}".format(scope_label, int(postgres.get('cluster_unlocked', 0))))
|
||||
|
||||
metrics.append("# HELP patroni_failsafe_mode_is_active Value is 1 if failsafe mode is active, 0 if inactive.")
|
||||
metrics.append("# TYPE patroni_failsafe_mode_is_active gauge")
|
||||
metrics.append("patroni_failsafe_mode_is_active{0} {1}"
|
||||
.format(labels, int(postgres.get('failsafe_mode_is_active', 0))))
|
||||
.format(scope_label, int(postgres.get('failsafe_mode_is_active', 0))))
|
||||
|
||||
metrics.append("# HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise.")
|
||||
metrics.append("# TYPE patroni_postgres_timeline counter")
|
||||
metrics.append("patroni_postgres_timeline{0} {1}".format(labels, postgres.get('timeline', 0)))
|
||||
metrics.append("patroni_postgres_timeline{0} {1}".format(scope_label, postgres.get('timeline', 0)))
|
||||
|
||||
metrics.append("# HELP patroni_dcs_last_seen Epoch timestamp when DCS was last contacted successfully"
|
||||
" by Patroni.")
|
||||
metrics.append("# TYPE patroni_dcs_last_seen gauge")
|
||||
metrics.append("patroni_dcs_last_seen{0} {1}".format(labels, postgres.get('dcs_last_seen', 0)))
|
||||
metrics.append("patroni_dcs_last_seen{0} {1}".format(scope_label, postgres.get('dcs_last_seen', 0)))
|
||||
|
||||
metrics.append("# HELP patroni_pending_restart Value is 1 if the node needs a restart, 0 otherwise.")
|
||||
metrics.append("# TYPE patroni_pending_restart gauge")
|
||||
metrics.append("patroni_pending_restart{0} {1}"
|
||||
.format(labels, int(patroni.postgresql.pending_restart)))
|
||||
.format(scope_label, int(patroni.postgresql.pending_restart)))
|
||||
|
||||
metrics.append("# HELP patroni_is_paused Value is 1 if auto failover is disabled, 0 otherwise.")
|
||||
metrics.append("# TYPE patroni_is_paused gauge")
|
||||
metrics.append("patroni_is_paused{0} {1}".format(labels, int(postgres.get('pause', 0))))
|
||||
metrics.append("patroni_is_paused{0} {1}".format(scope_label, int(postgres.get('pause', 0))))
|
||||
|
||||
self.write_response(200, '\n'.join(metrics) + '\n', content_type='text/plain')
|
||||
|
||||
@@ -776,6 +770,44 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
self.server.patroni.api_sigterm()
|
||||
self.write_response(202, 'shutdown scheduled')
|
||||
|
||||
@staticmethod
|
||||
def parse_schedule(schedule: str,
|
||||
action: str) -> Tuple[Union[int, None], Union[str, None], Union[datetime.datetime, None]]:
|
||||
"""Parse the given *schedule* and validate it.
|
||||
|
||||
:param schedule: a string representing a timestamp, e.g. ``2023-04-14T20:27:00+00:00``.
|
||||
:param action: the action to be scheduled (``restart``, ``switchover``, or ``failover``).
|
||||
|
||||
:returns: a tuple composed of 3 items:
|
||||
|
||||
* Suggested HTTP status code for a response:
|
||||
|
||||
* ``None``: if no issue was faced while parsing, leaving it up to the caller to decide the status; or
|
||||
* ``400``: if no timezone information could be found in *schedule*; or
|
||||
* ``422``: if *schedule* is invalid -- in the past or not parsable.
|
||||
|
||||
* An error message, if any error is faced, otherwise ``None``;
|
||||
* Parsed *schedule*, if able to parse, otherwise ``None``.
|
||||
|
||||
"""
|
||||
error = None
|
||||
scheduled_at = None
|
||||
try:
|
||||
scheduled_at = dateutil.parser.parse(schedule)
|
||||
if scheduled_at.tzinfo is None:
|
||||
error = 'Timezone information is mandatory for the scheduled {0}'.format(action)
|
||||
status_code = 400
|
||||
elif scheduled_at < datetime.datetime.now(tzutc):
|
||||
error = 'Cannot schedule {0} in the past'.format(action)
|
||||
status_code = 422
|
||||
else:
|
||||
status_code = None
|
||||
except (ValueError, TypeError):
|
||||
logger.exception('Invalid scheduled %s time: %s', action, schedule)
|
||||
error = 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601'
|
||||
status_code = 422
|
||||
return status_code, error, scheduled_at
|
||||
|
||||
@check_access
|
||||
def do_POST_restart(self) -> None:
|
||||
"""Handle a ``POST`` request to ``/restart`` path.
|
||||
@@ -831,9 +863,9 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
|
||||
for k in request:
|
||||
if k == 'schedule':
|
||||
parse_result, request[k] = parse_schedule(request[k])
|
||||
if parse_result:
|
||||
data, status_code = parse_result.value[0], parse_result.value[1]
|
||||
(_, data, request[k]) = self.parse_schedule(request[k], "restart")
|
||||
if _:
|
||||
status_code = _
|
||||
break
|
||||
elif k == 'role':
|
||||
if request[k] not in ('master', 'primary', 'replica'):
|
||||
@@ -983,6 +1015,39 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
logger.debug('Exception occurred during polling %s result: %s', action, e)
|
||||
return 503, action.title() + ' status unknown'
|
||||
|
||||
def is_failover_possible(self, cluster: Cluster, leader: Optional[str], candidate: Optional[str],
|
||||
action: str) -> Optional[str]:
|
||||
"""Checks whether there are nodes that could take over after demoting the primary.
|
||||
|
||||
:param cluster: the Patroni cluster.
|
||||
:param leader: name of the current Patroni leader.
|
||||
:param candidate: name of the Patroni node to be promoted.
|
||||
:param action: the action to be performed (``switchover`` or ``failover``).
|
||||
|
||||
:returns: a string with the error message or ``None`` if good nodes are found.
|
||||
"""
|
||||
is_synchronous_mode = self.server.patroni.config.get_global_config(cluster).is_synchronous_mode
|
||||
if leader and (not cluster.leader or cluster.leader.name != leader):
|
||||
return 'leader name does not match'
|
||||
if candidate:
|
||||
if action == 'switchover' and is_synchronous_mode and not cluster.sync.matches(candidate):
|
||||
return 'candidate name does not match with sync_standby'
|
||||
members = [m for m in cluster.members if m.name == candidate]
|
||||
if not members:
|
||||
return 'candidate does not exists'
|
||||
elif is_synchronous_mode:
|
||||
members = [m for m in cluster.members if cluster.sync.matches(m.name)]
|
||||
if not members:
|
||||
return action + ' is not possible: can not find sync_standby'
|
||||
else:
|
||||
members = [m for m in cluster.members if not cluster.leader or m.name != cluster.leader.name and m.api_url]
|
||||
if not members:
|
||||
return action + ' is not possible: cluster does not have members except leader'
|
||||
for st in self.server.patroni.ha.fetch_nodes_statuses(members):
|
||||
if st.failover_limitation() is None:
|
||||
return None
|
||||
return action + ' is not possible: no good candidates have been found'
|
||||
|
||||
@check_access
|
||||
def do_POST_failover(self, action: str = 'failover') -> None:
|
||||
"""Handle a ``POST`` request to ``/failover`` path.
|
||||
@@ -1002,8 +1067,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
* ``412``: if operation is not possible;
|
||||
* ``503``: if unable to register the operation to the DCS;
|
||||
* HTTP status returned by :func:`parse_schedule`, if any error was observed while parsing the schedule;
|
||||
* HTTP status returned by :func:`poll_failover_result` if the operation has been processed immediately;
|
||||
* ``400``: if none of the above applies.
|
||||
* HTTP status returned by :func:`poll_failover_result` if the operation has been processed immediately.
|
||||
|
||||
.. note::
|
||||
If unable to parse the request body, then the request is silently discarded.
|
||||
@@ -1011,6 +1075,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
:param action: the action to be performed (``switchover`` or ``failover``).
|
||||
"""
|
||||
request = self._read_json_content()
|
||||
(status_code, data) = (400, '')
|
||||
if not request:
|
||||
return
|
||||
|
||||
@@ -1023,15 +1088,26 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
logger.info("received %s request with leader=%s candidate=%s scheduled_at=%s",
|
||||
action, leader, candidate, scheduled_at)
|
||||
|
||||
manual_failover = ManualFailover(action, cluster, leader, candidate, scheduled_at,
|
||||
global_config.is_paused, global_config.is_synchronous_mode,
|
||||
self.server.patroni)
|
||||
data, status_code = manual_failover.run_precheck().value
|
||||
if action == 'failover' and not candidate:
|
||||
data = 'Failover could be performed only to a specific candidate'
|
||||
elif action == 'switchover' and not leader:
|
||||
data = 'Switchover could be performed only from a specific leader'
|
||||
|
||||
if not data and scheduled_at:
|
||||
parse_result, scheduled_at = manual_failover.parse_scheduled()
|
||||
if parse_result:
|
||||
data, status_code = parse_result.value[0], parse_result.value[1]
|
||||
if not leader:
|
||||
data = 'Scheduled {0} is possible only from a specific leader'.format(action)
|
||||
if not data and global_config.is_paused:
|
||||
data = "Can't schedule {0} in the paused state".format(action)
|
||||
if not data:
|
||||
(status_code, data, scheduled_at) = self.parse_schedule(scheduled_at, action)
|
||||
|
||||
if not data and global_config.is_paused and not candidate:
|
||||
data = action.title() + ' is possible only to a specific candidate in a paused state'
|
||||
|
||||
if not data and not scheduled_at:
|
||||
data = self.is_failover_possible(cluster, leader, candidate, action)
|
||||
if data:
|
||||
status_code = 412
|
||||
|
||||
if not data:
|
||||
if self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
|
||||
@@ -1043,12 +1119,14 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name,
|
||||
candidate, action)
|
||||
else:
|
||||
data = 'failed to write failover key into DCS'
|
||||
data = 'failed to write {0} key into DCS'.format(action)
|
||||
status_code = 503
|
||||
|
||||
status_code = status_code or 400
|
||||
self.write_response(status_code, data.format(action=action, leader=leader, candidate=candidate,
|
||||
cluster_name=self.server.patroni.postgresql.scope))
|
||||
# pyright thinks ``status_code`` can be ``None`` because ``parse_schedule`` call may return ``None``. However,
|
||||
# if that's the case, ``status_code`` will be overwritten somewhere between ``parse_schedule`` and
|
||||
# ``write_response`` calls.
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
assert isinstance(status_code, int)
|
||||
self.write_response(status_code, data)
|
||||
|
||||
def do_POST_switchover(self) -> None:
|
||||
"""Handle a ``POST`` request to ``/switchover`` path.
|
||||
@@ -1105,18 +1183,20 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
self.command = mname
|
||||
return ret
|
||||
|
||||
def query(self, sql: str, *params: Any, retry: bool = False) -> List[Tuple[Any, ...]]:
|
||||
"""Execute *sql* query with *params* and optionally return results.
|
||||
def query(self, sql: str, *params: Any, **kwargs: Any) -> List[Tuple[Any, ...]]:
|
||||
"""Execute *sql* query with *params*.
|
||||
|
||||
:param sql: the SQL statement to be run.
|
||||
:param params: positional arguments to call :func:`RestApiServer.query` with.
|
||||
:param retry: whether the query should be retried upon failure or given up immediately.
|
||||
:param kwargs: can contain the key ``retry``. If the key is present its value should be a :class:`bool` which
|
||||
indicates whether the query should be retried upon failure or given up immediately.
|
||||
|
||||
:returns: a list of rows that were fetched from the database.
|
||||
"""
|
||||
if not retry:
|
||||
if not kwargs.get('retry', False):
|
||||
return self.server.query(sql, *params)
|
||||
return Retry(delay=1, retry_exceptions=PostgresConnectionException)(self.server.query, sql, *params)
|
||||
retry = Retry(delay=1, retry_exceptions=PostgresConnectionException)
|
||||
return retry(self.server.query, sql, *params)
|
||||
|
||||
def get_postgresql_status(self, retry: bool = False) -> Dict[str, Any]:
|
||||
"""Builds an object representing a status of "postgres".
|
||||
@@ -1182,8 +1262,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
" application_name, client_addr, w.state, sync_state, sync_priority"
|
||||
" FROM pg_catalog.pg_stat_get_wal_senders() w, pg_catalog.pg_stat_get_activity(pid)) AS ri")
|
||||
|
||||
row = self.query(stmt.format(postgresql.wal_name, postgresql.lsn_name,
|
||||
postgresql.wal_flush), retry=retry)[0]
|
||||
row = self.query(stmt.format(postgresql.wal_name, postgresql.lsn_name), retry=retry)[0]
|
||||
|
||||
result = {
|
||||
'state': postgresql.state,
|
||||
'postmaster_start_time': row[0],
|
||||
@@ -1288,10 +1368,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
|
||||
self.daemon = True
|
||||
|
||||
def query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]:
|
||||
"""Execute *sql* query with *params* and optionally return results.
|
||||
|
||||
.. note::
|
||||
Prefer to use own connection to postgres and fallback to ``heartbeat`` when own isn't available.
|
||||
"""Execute *sql* query with *params*.
|
||||
|
||||
:param sql: the SQL statement to be run.
|
||||
:param params: positional arguments to be used as parameters for *sql*.
|
||||
@@ -1302,21 +1379,15 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
|
||||
:class:`psycopg.Error`: if had issues while executing *sql*.
|
||||
:class:`~patroni.exceptions.PostgresConnectionException`: if had issues while connecting to the database.
|
||||
"""
|
||||
# We first try to get a heartbeat connection because it is always required for the main thread.
|
||||
cursor = None
|
||||
try:
|
||||
heartbeat_connection = self.patroni.postgresql.connection_pool.get('heartbeat')
|
||||
heartbeat_connection.get() # try to open psycopg connection to postgres
|
||||
except psycopg.Error as exc:
|
||||
raise PostgresConnectionException('connection problems') from exc
|
||||
|
||||
try:
|
||||
connection = self.patroni.postgresql.connection_pool.get('restapi')
|
||||
connection.get() # try to open psycopg connection to postgres
|
||||
except psycopg.Error:
|
||||
logger.debug('restapi connection to postgres is not available')
|
||||
connection = heartbeat_connection
|
||||
|
||||
return connection.query(sql, *params)
|
||||
with self.patroni.postgresql.connection().cursor() as cursor:
|
||||
cursor.execute(sql.encode('utf-8'), params)
|
||||
return [r for r in cursor]
|
||||
except psycopg.Error as e:
|
||||
if cursor and cursor.connection.closed == 0:
|
||||
raise e
|
||||
raise PostgresConnectionException('connection problems')
|
||||
|
||||
@staticmethod
|
||||
def _set_fd_cloexec(fd: socket.socket) -> None:
|
||||
@@ -1482,7 +1553,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
|
||||
:param listen: IP and port to bind REST API to. It should be a string in the format ``host:port``, where
|
||||
``host`` can be a hostname or IP address. It is the value of ``restapi.listen`` setting.
|
||||
:param ssl_options: dictionary that may contain the following keys, depending on what has been configured in
|
||||
``restapi`` section:
|
||||
``restapi` section:
|
||||
|
||||
* ``certfile``: path to PEM certificate. If given, will start in HTTPS mode;
|
||||
* ``keyfile``: path to key of ``certfile``;
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
Provides a case insensitive :class:`dict` and :class:`set` object types.
|
||||
"""
|
||||
from collections import OrderedDict
|
||||
from typing import Any, Collection, Dict, Iterator, KeysView, MutableMapping, MutableSet, Optional
|
||||
from typing import Any, Collection, Dict, Iterator, MutableMapping, MutableSet, Optional
|
||||
|
||||
|
||||
class CaseInsensitiveSet(MutableSet[str]):
|
||||
@@ -187,13 +187,6 @@ class CaseInsensitiveDict(MutableMapping[str, Any]):
|
||||
"""
|
||||
return CaseInsensitiveDict({v[0]: v[1] for v in self._values.values()})
|
||||
|
||||
def keys(self) -> KeysView[str]:
|
||||
"""Return a new view of the dict's keys.
|
||||
|
||||
:returns: a set-like object providing a view on the dict's keys
|
||||
"""
|
||||
return self._values.keys()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
"""Get a string representation of the dict.
|
||||
|
||||
|
||||
+69
-347
@@ -1,4 +1,3 @@
|
||||
"""Facilities related to Patroni configuration."""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -16,6 +15,7 @@ from .dcs import ClusterConfig, Cluster
|
||||
from .exceptions import ConfigParseError
|
||||
from .file_perm import pg_perm
|
||||
from .postgresql.config import ConfigHandler
|
||||
from .validator import IntValidator
|
||||
from .utils import deep_compare, parse_bool, parse_int, patch_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -36,162 +36,121 @@ _AUTH_ALLOWED_PARAMETERS = (
|
||||
|
||||
|
||||
def default_validator(conf: Dict[str, Any]) -> List[str]:
|
||||
"""Ensure *conf* is not empty.
|
||||
|
||||
Designed to be used as default validator for :class:`Config` objects, if no specific validator is provided.
|
||||
|
||||
:param conf: configuration to be validated.
|
||||
|
||||
:returns: an empty list -- :class:`Config` expects the validator to return a list of 0 or more issues found while
|
||||
validating the configuration.
|
||||
|
||||
:raises:
|
||||
:class:`ConfigParseError`: if *conf* is empty.
|
||||
"""
|
||||
if not conf:
|
||||
raise ConfigParseError("Config is empty.")
|
||||
return []
|
||||
|
||||
|
||||
class GlobalConfig(object):
|
||||
"""A class that wraps global configuration and provides convenient methods to access/check values.
|
||||
|
||||
It is instantiated either by calling :func:`get_global_config` or :meth:`Config.get_global_config`, which picks
|
||||
either a configuration from provided :class:`Cluster` object (the most up-to-date) or from the
|
||||
local cache if :class:`ClusterConfig` is not initialized or doesn't have a valid config.
|
||||
"""A class that wrapps global configuration and provides convinient methods to access/check values.
|
||||
|
||||
It is instantiated by calling :func:`Config.global_config` method which picks either a
|
||||
configuration from provided :class:`Cluster` object (the most up-to-date) or from the
|
||||
local cache if :class::`ClusterConfig` is not initialized or doesn't have a valid config.
|
||||
"""
|
||||
|
||||
def __init__(self, config: Dict[str, Any]) -> None:
|
||||
"""Initialize :class:`GlobalConfig` object with given *config*.
|
||||
"""Initialize :class:`GlobalConfig` object.
|
||||
|
||||
:param config: current configuration either from
|
||||
:class:`ClusterConfig` or from :func:`Config.dynamic_configuration`.
|
||||
:class:`ClusterConfig` or from :class:`Config.dynamic_configuration`
|
||||
"""
|
||||
self.__config = config
|
||||
|
||||
def get(self, name: str) -> Any:
|
||||
"""Gets global configuration value by *name*.
|
||||
"""Gets global configuration value by name.
|
||||
|
||||
:param name: parameter name.
|
||||
|
||||
:returns: configuration value or ``None`` if it is missing.
|
||||
:param name: parameter name
|
||||
:returns: configuration value or `None` if it is missing
|
||||
"""
|
||||
return self.__config.get(name)
|
||||
|
||||
def check_mode(self, mode: str) -> bool:
|
||||
"""Checks whether the certain parameter is enabled.
|
||||
|
||||
:param mode: parameter name, e.g. ``synchronous_mode``, ``failsafe_mode``, ``pause``, ``check_timeline``, and
|
||||
so on.
|
||||
|
||||
:returns: ``True`` if parameter *mode* is enabled in the global configuration.
|
||||
:param mode: parameter name could be: synchronous_mode, failsafe_mode, pause, check_timeline, and so on
|
||||
:returns: `True` if *mode* is enabled in the global configuration.
|
||||
"""
|
||||
return bool(parse_bool(self.__config.get(mode)))
|
||||
|
||||
@property
|
||||
def is_paused(self) -> bool:
|
||||
"""``True`` if cluster is in maintenance mode."""
|
||||
""":returns: `True` if cluster is in maintenance mode."""
|
||||
return self.check_mode('pause')
|
||||
|
||||
@property
|
||||
def is_synchronous_mode(self) -> bool:
|
||||
"""``True`` if synchronous replication is requested."""
|
||||
""":returns: `True` if synchronous replication is requested."""
|
||||
return self.check_mode('synchronous_mode')
|
||||
|
||||
@property
|
||||
def is_synchronous_mode_strict(self) -> bool:
|
||||
"""``True`` if at least one synchronous node is required."""
|
||||
""":returns: `True` if at least one synchronous node is required."""
|
||||
return self.check_mode('synchronous_mode_strict')
|
||||
|
||||
def get_standby_cluster_config(self) -> Union[Dict[str, Any], Any]:
|
||||
"""Get ``standby_cluster`` configuration.
|
||||
|
||||
:returns: a copy of ``standby_cluster`` configuration.
|
||||
"""
|
||||
""":returns: "standby_cluster" configuration."""
|
||||
return deepcopy(self.get('standby_cluster'))
|
||||
|
||||
@property
|
||||
def is_standby_cluster(self) -> bool:
|
||||
"""``True`` if global configuration has a valid ``standby_cluster`` section."""
|
||||
""":returns: `True` if global configuration has a valid "standby_cluster" section."""
|
||||
config = self.get_standby_cluster_config()
|
||||
return isinstance(config, dict) and\
|
||||
bool(config.get('host') or config.get('port') or config.get('restore_command'))
|
||||
|
||||
def get_int(self, name: str, default: int = 0) -> int:
|
||||
"""Gets current value of *name* from the global configuration and try to return it as :class:`int`.
|
||||
"""Gets current value from the global configuration and trying to return it as int.
|
||||
|
||||
:param name: name of the parameter.
|
||||
:param default: default value if *name* is not in the configuration or invalid.
|
||||
|
||||
:returns: currently configured value of *name* from the global configuration or *default* if it is not set or
|
||||
invalid.
|
||||
:param name: name of the parameter
|
||||
:param default: default value if *name* is not in the configuration or invalid
|
||||
:returns: currently configured value from the global configuration or *default* if it is not set or invalid.
|
||||
"""
|
||||
ret = parse_int(self.get(name))
|
||||
return default if ret is None else ret
|
||||
|
||||
@property
|
||||
def min_synchronous_nodes(self) -> int:
|
||||
"""The minimal number of synchronous nodes based on whether ``synchronous_mode_strict`` is enabled or not."""
|
||||
""":returns: the minimal number of synchronous nodes based on whether strict mode is requested or not."""
|
||||
return 1 if self.is_synchronous_mode_strict else 0
|
||||
|
||||
@property
|
||||
def synchronous_node_count(self) -> int:
|
||||
"""Currently configured value of ``synchronous_node_count`` from the global configuration.
|
||||
|
||||
Assume ``1`` if it is not set or invalid.
|
||||
"""
|
||||
""":returns: currently configured value from the global configuration or 1 if it is not set or invalid."""
|
||||
return max(self.get_int('synchronous_node_count', 1), self.min_synchronous_nodes)
|
||||
|
||||
@property
|
||||
def maximum_lag_on_failover(self) -> int:
|
||||
"""Currently configured value of ``maximum_lag_on_failover`` from the global configuration.
|
||||
|
||||
Assume ``1048576`` if it is not set or invalid.
|
||||
"""
|
||||
""":returns: currently configured value from the global configuration or 1048576 if it is not set or invalid."""
|
||||
return self.get_int('maximum_lag_on_failover', 1048576)
|
||||
|
||||
@property
|
||||
def maximum_lag_on_syncnode(self) -> int:
|
||||
"""Currently configured value of ``maximum_lag_on_syncnode`` from the global configuration.
|
||||
|
||||
Assume ``-1`` if it is not set or invalid.
|
||||
"""
|
||||
""":returns: currently configured value from the global configuration or -1 if it is not set or invalid."""
|
||||
return self.get_int('maximum_lag_on_syncnode', -1)
|
||||
|
||||
@property
|
||||
def primary_start_timeout(self) -> int:
|
||||
"""Currently configured value of ``primary_start_timeout`` from the global configuration.
|
||||
|
||||
Assume ``300`` if it is not set or invalid.
|
||||
|
||||
.. note::
|
||||
``master_start_timeout`` is still supported to keep backward compatibility.
|
||||
"""
|
||||
""":returns: currently configured value from the global configuration or 300 if it is not set or invalid."""
|
||||
default = 300
|
||||
return self.get_int('primary_start_timeout', default)\
|
||||
if 'primary_start_timeout' in self.__config else self.get_int('master_start_timeout', default)
|
||||
|
||||
@property
|
||||
def primary_stop_timeout(self) -> int:
|
||||
"""Currently configured value of ``primary_stop_timeout`` from the global configuration.
|
||||
|
||||
Assume ``0`` if it is not set or invalid.
|
||||
|
||||
.. note::
|
||||
``master_stop_timeout`` is still supported to keep backward compatibility.
|
||||
"""
|
||||
""":returns: currently configured value from the global configuration or 300 if it is not set or invalid."""
|
||||
default = 0
|
||||
return self.get_int('primary_stop_timeout', default)\
|
||||
if 'primary_stop_timeout' in self.__config else self.get_int('master_stop_timeout', default)
|
||||
|
||||
|
||||
def get_global_config(cluster: Optional[Cluster], default: Optional[Dict[str, Any]] = None) -> GlobalConfig:
|
||||
def get_global_config(cluster: Union[Cluster, None], default: Optional[Dict[str, Any]] = None) -> GlobalConfig:
|
||||
"""Instantiates :class:`GlobalConfig` based on the input.
|
||||
|
||||
:param cluster: the currently known cluster state from DCS.
|
||||
:param default: default configuration, which will be used if there is no valid *cluster.config*.
|
||||
|
||||
:returns: :class:`GlobalConfig` object.
|
||||
:param cluster: the currently known cluster state from DCS
|
||||
:param default: default configuration, which will be used if there is no valid *cluster.config*
|
||||
:returns: :class:`GlobalConfig` object
|
||||
"""
|
||||
# Try to protect from the case when DCS was wiped out
|
||||
if cluster and cluster.config and cluster.config.modify_version:
|
||||
@@ -202,29 +161,23 @@ def get_global_config(cluster: Optional[Cluster], default: Optional[Dict[str, An
|
||||
|
||||
|
||||
class Config(object):
|
||||
"""Handle Patroni configuration.
|
||||
|
||||
"""
|
||||
This class is responsible for:
|
||||
|
||||
1) Building and giving access to ``effective_configuration`` from:
|
||||
1) Building and giving access to `effective_configuration` from:
|
||||
* `Config.__DEFAULT_CONFIG` -- some sane default values
|
||||
* `dynamic_configuration` -- configuration stored in DCS
|
||||
* `local_configuration` -- configuration from `config.yml` or environment
|
||||
|
||||
* ``Config.__DEFAULT_CONFIG`` -- some sane default values;
|
||||
* ``dynamic_configuration`` -- configuration stored in DCS;
|
||||
* ``local_configuration`` -- configuration from `config.yml` or environment.
|
||||
|
||||
2) Saving and loading ``dynamic_configuration`` into 'patroni.dynamic.json' file
|
||||
2) Saving and loading `dynamic_configuration` into 'patroni.dynamic.json' file
|
||||
located in local_configuration['postgresql']['data_dir'] directory.
|
||||
This is necessary to be able to restore ``dynamic_configuration``
|
||||
if DCS was accidentally wiped.
|
||||
This is necessary to be able to restore `dynamic_configuration`
|
||||
if DCS was accidentally wiped
|
||||
|
||||
3) Loading of configuration file in the old format and converting it into new format.
|
||||
3) Loading of configuration file in the old format and converting it into new format
|
||||
|
||||
4) Mimicking some ``dict`` interfaces to make it possible
|
||||
to work with it as with the old ``config`` object.
|
||||
|
||||
:cvar PATRONI_CONFIG_VARIABLE: name of the environment variable that can be used to load Patroni configuration from.
|
||||
:cvar __CACHE_FILENAME: name of the file used to cache dynamic configuration under Postgres data directory.
|
||||
:cvar __DEFAULT_CONFIG: default configuration values for some Patroni settings.
|
||||
4) Mimicking some of the `dict` interfaces to make it possible
|
||||
to work with it as with the old `config` object.
|
||||
"""
|
||||
|
||||
PATRONI_CONFIG_VARIABLE = PATRONI_ENV_PREFIX + 'CONFIGURATION'
|
||||
@@ -242,38 +195,21 @@ class Config(object):
|
||||
'recovery_min_apply_delay': ''
|
||||
},
|
||||
'postgresql': {
|
||||
'bin_dir': '',
|
||||
'use_slots': True,
|
||||
'parameters': CaseInsensitiveDict({p: v[0] for p, v in ConfigHandler.CMDLINE_OPTIONS.items()
|
||||
if v[0] is not None and p not in ('wal_keep_segments', 'wal_keep_size')})
|
||||
if p not in ('wal_keep_segments', 'wal_keep_size')})
|
||||
}
|
||||
}
|
||||
|
||||
def __init__(self, configfile: str,
|
||||
validator: Optional[Callable[[Dict[str, Any]], List[str]]] = default_validator) -> None:
|
||||
"""Create a new instance of :class:`Config` and validate the loaded configuration using *validator*.
|
||||
|
||||
.. note::
|
||||
Patroni will read configuration from these locations in this order:
|
||||
|
||||
* file or directory path passed as command-line argument (*configfile*), if it exists and the file or
|
||||
files found in the directory can be parsed (see :meth:`~Config._load_config_path`), otherwise
|
||||
* YAML file passed via the environment variable (see :attr:`PATRONI_CONFIG_VARIABLE`), if the referenced
|
||||
file exists and can be parsed, otherwise
|
||||
* from configuration values defined as environment variables, see
|
||||
:meth:`~Config._build_environment_configuration`.
|
||||
|
||||
:param configfile: path to Patroni configuration file.
|
||||
:param validator: function used to validate Patroni configuration. It should receive a dictionary which
|
||||
represents Patroni configuration, and return a list of zero or more error messages based on validation.
|
||||
|
||||
:raises:
|
||||
:class:`ConfigParseError`: if any issue is reported by *validator*.
|
||||
"""
|
||||
self._modify_version = -1
|
||||
self._dynamic_configuration = {}
|
||||
|
||||
self.__environment_configuration = self._build_environment_configuration()
|
||||
|
||||
# Patroni reads the configuration from the command-line argument if it exists, otherwise from the environment
|
||||
self._config_file = configfile if configfile and os.path.exists(configfile) else None
|
||||
if self._config_file:
|
||||
self._local_configuration = self._load_config_file()
|
||||
@@ -294,43 +230,17 @@ class Config(object):
|
||||
self._cache_needs_saving = False
|
||||
|
||||
@property
|
||||
def config_file(self) -> Optional[str]:
|
||||
"""Path to Patroni configuration file, if any, else ``None``."""
|
||||
def config_file(self) -> Union[str, None]:
|
||||
return self._config_file
|
||||
|
||||
@property
|
||||
def dynamic_configuration(self) -> Dict[str, Any]:
|
||||
"""Deep copy of cached Patroni dynamic configuration."""
|
||||
return deepcopy(self._dynamic_configuration)
|
||||
|
||||
@property
|
||||
def local_configuration(self) -> Dict[str, Any]:
|
||||
"""Deep copy of cached Patroni local configuration.
|
||||
|
||||
:returns: copy of :attr:`~Config._local_configuration`
|
||||
"""
|
||||
return deepcopy(dict(self._local_configuration))
|
||||
|
||||
@classmethod
|
||||
def get_default_config(cls) -> Dict[str, Any]:
|
||||
"""Deep copy default configuration.
|
||||
|
||||
:returns: copy of :attr:`~Config.__DEFAULT_CONFIG`
|
||||
"""
|
||||
return deepcopy(cls.__DEFAULT_CONFIG)
|
||||
|
||||
def _load_config_path(self, path: str) -> Dict[str, Any]:
|
||||
"""Load Patroni configuration file(s) from *path*.
|
||||
|
||||
If *path* is a file, load the yml file pointed to by *path*.
|
||||
If *path* is a directory, load all yml files in that directory in alphabetical order.
|
||||
|
||||
:param path: path to either an YAML configuration file, or to a folder containing YAML configuration files.
|
||||
|
||||
:returns: configuration after reading the configuration file(s) from *path*.
|
||||
|
||||
:raises:
|
||||
:class:`ConfigParseError`: if *path* is invalid.
|
||||
"""
|
||||
If path is a file, loads the yml file pointed to by path.
|
||||
If path is a directory, loads all yml files in that directory in alphabetical order
|
||||
"""
|
||||
if os.path.isfile(path):
|
||||
files = [path]
|
||||
@@ -349,18 +259,14 @@ class Config(object):
|
||||
return overall_config
|
||||
|
||||
def _load_config_file(self) -> Dict[str, Any]:
|
||||
"""Load configuration file(s) from filesystem and apply values which were set via environment variables.
|
||||
|
||||
:returns: final configuration after merging configuration file(s) and environment variables.
|
||||
"""
|
||||
"""Loads config.yaml from filesystem and applies some values which were set via ENV"""
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
assert self.config_file is not None
|
||||
config = self._load_config_path(self.config_file)
|
||||
assert self._config_file is not None
|
||||
config = self._load_config_path(self._config_file)
|
||||
patch_config(config, self.__environment_configuration)
|
||||
return config
|
||||
|
||||
def _load_cache(self) -> None:
|
||||
"""Load dynamic configuration from ``patroni.dynamic.json``."""
|
||||
if os.path.isfile(self._cache_file):
|
||||
try:
|
||||
with open(self._cache_file) as f:
|
||||
@@ -369,12 +275,6 @@ class Config(object):
|
||||
logger.exception('Exception when loading file: %s', self._cache_file)
|
||||
|
||||
def save_cache(self) -> None:
|
||||
"""Save dynamic configuration to ``patroni.dynamic.json`` under Postgres data directory.
|
||||
|
||||
.. note::
|
||||
``patroni.dynamic.jsonXXXXXX`` is created as a temporary file and than renamed to ``patroni.dynamic.json``,
|
||||
where ``XXXXXX`` is a random suffix.
|
||||
"""
|
||||
if self._cache_needs_saving:
|
||||
tmpfile = fd = None
|
||||
try:
|
||||
@@ -401,16 +301,9 @@ class Config(object):
|
||||
|
||||
# configuration could be either ClusterConfig or dict
|
||||
def set_dynamic_configuration(self, configuration: Union[ClusterConfig, Dict[str, Any]]) -> bool:
|
||||
"""Set dynamic configuration values with given *configuration*.
|
||||
|
||||
:param configuration: new dynamic configuration values. Supports :class:`dict` for backward compatibility.
|
||||
|
||||
:returns: ``True`` if changes have been detected between current dynamic configuration and the new dynamic
|
||||
*configuration*, ``False`` otherwise.
|
||||
"""
|
||||
if isinstance(configuration, ClusterConfig):
|
||||
if self._modify_version == configuration.modify_version:
|
||||
return False # If the version didn't change there is nothing to do
|
||||
return False # If the version didn't changed there is nothing to do
|
||||
self._modify_version = configuration.modify_version
|
||||
configuration = configuration.data
|
||||
|
||||
@@ -426,14 +319,6 @@ class Config(object):
|
||||
return False
|
||||
|
||||
def reload_local_configuration(self) -> Optional[bool]:
|
||||
"""Reload configuration values from the configuration file(s).
|
||||
|
||||
.. note::
|
||||
Designed to be used when user applies changes to configuration file(s), so Patroni can use the new values
|
||||
with a reload instead of a restart.
|
||||
|
||||
:returns: ``True`` if changes have been detected between current local configuration
|
||||
"""
|
||||
if self.config_file:
|
||||
try:
|
||||
configuration = self._load_config_file()
|
||||
@@ -449,46 +334,15 @@ class Config(object):
|
||||
|
||||
@staticmethod
|
||||
def _process_postgresql_parameters(parameters: Dict[str, Any], is_local: bool = False) -> Dict[str, Any]:
|
||||
"""Process Postgres *parameters*.
|
||||
|
||||
.. note::
|
||||
If *is_local* configuration discard any setting from *parameters* that is listed under
|
||||
:attr:`~patroni.postgresql.config.ConfigHandler.CMDLINE_OPTIONS` as those are supposed to be set only
|
||||
through dynamic configuration.
|
||||
|
||||
When setting parameters from :attr:`~patroni.postgresql.config.ConfigHandler.CMDLINE_OPTIONS` through
|
||||
dynamic configuration their value will be validated as per the validator defined in that very same
|
||||
attribute entry. If the given value cannot be validated, a warning will be logged and the default value of
|
||||
the GUC will be used instead.
|
||||
|
||||
Some parameters from :attr:`~patroni.postgresql.config.ConfigHandler.CMDLINE_OPTIONS` cannot be set even if
|
||||
not *is_local* configuration:
|
||||
|
||||
* ``listen_addresses``: inferred from ``postgresql.listen`` local configuration or from
|
||||
``PATRONI_POSTGRESQL_LISTEN`` environment variable;
|
||||
* ``port``: inferred from ``postgresql.listen`` local configuration or from
|
||||
``PATRONI_POSTGRESQL_LISTEN`` environment variable;
|
||||
* ``cluster_name``: set through ``scope`` local configuration or through ``PATRONI_SCOPE`` environment
|
||||
variable;
|
||||
* ``hot_standby``: always enabled;
|
||||
* ``wal_log_hints``: always enabled.
|
||||
|
||||
:param parameters: Postgres parameters to be processed. Should be the parsed YAML value of
|
||||
``postgresql.parameters`` configuration, either from local or from dynamic configuration.
|
||||
|
||||
:param is_local: should be ``True`` if *parameters* refers to local configuration, or ``False`` if *parameters*
|
||||
refers to dynamic configuration.
|
||||
|
||||
:returns: new value for ``postgresql.parameters`` after processing and validating *parameters*.
|
||||
"""
|
||||
pg_params: Dict[str, Any] = {}
|
||||
|
||||
for name, value in (parameters or {}).items():
|
||||
if name not in ConfigHandler.CMDLINE_OPTIONS:
|
||||
pg_params[name] = value
|
||||
elif not is_local:
|
||||
if ConfigHandler.CMDLINE_OPTIONS[name][1](value):
|
||||
pg_params[name] = value
|
||||
validator = ConfigHandler.CMDLINE_OPTIONS[name][1]
|
||||
if validator(value):
|
||||
pg_params[name] = int(value) if isinstance(validator, IntValidator) else value
|
||||
else:
|
||||
logger.warning("postgresql parameter %s=%s failed validation, defaulting to %s",
|
||||
name, value, ConfigHandler.CMDLINE_OPTIONS[name][0])
|
||||
@@ -496,33 +350,7 @@ class Config(object):
|
||||
return pg_params
|
||||
|
||||
def _safe_copy_dynamic_configuration(self, dynamic_configuration: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Create a copy of *dynamic_configuration*.
|
||||
|
||||
Merge *dynamic_configuration* with :attr:`__DEFAULT_CONFIG` (*dynamic_configuration* takes precedence), and
|
||||
process ``postgresql.parameters`` from *dynamic_configuration* through :func:`_process_postgresql_parameters`,
|
||||
if present.
|
||||
|
||||
.. note::
|
||||
The following settings are not allowed in ``postgresql`` section as they are intended to be local
|
||||
configuration, and are removed if present:
|
||||
|
||||
* ``connect_address``;
|
||||
* ``proxy_address``;
|
||||
* ``listen``;
|
||||
* ``config_dir``;
|
||||
* ``data_dir``;
|
||||
* ``pgpass``;
|
||||
* ``authentication``;
|
||||
|
||||
Besides that any setting present in *dynamic_configuration* but absent from :attr:`__DEFAULT_CONFIG` is
|
||||
discarded.
|
||||
|
||||
:param dynamic_configuration: Patroni dynamic configuration.
|
||||
|
||||
:returns: copy of *dynamic_configuration*, merged with default dynamic configuration and with some sanity checks
|
||||
performed over it.
|
||||
"""
|
||||
config = self.get_default_config()
|
||||
config = deepcopy(self.__DEFAULT_CONFIG)
|
||||
|
||||
for name, value in dynamic_configuration.items():
|
||||
if name == 'postgresql':
|
||||
@@ -542,25 +370,9 @@ class Config(object):
|
||||
|
||||
@staticmethod
|
||||
def _build_environment_configuration() -> Dict[str, Any]:
|
||||
"""Get local configuration settings that were specified through environment variables.
|
||||
|
||||
:returns: dictionary containing the found environment variables and their values, respecting the expected
|
||||
structure of Patroni configuration.
|
||||
"""
|
||||
ret: Dict[str, Any] = defaultdict(dict)
|
||||
|
||||
def _popenv(name: str) -> Optional[str]:
|
||||
"""Get value of environment variable *name*.
|
||||
|
||||
.. note::
|
||||
*name* is prefixed with :data:`~patroni.PATRONI_ENV_PREFIX` when searching in the environment.
|
||||
|
||||
Also, the corresponding environment variable is removed from the environment upon reading its value.
|
||||
|
||||
:param name: name of the environment variable.
|
||||
|
||||
:returns: value of *name*, if present in the environment, otherwise ``None``.
|
||||
"""
|
||||
def _popenv(name: str) -> Union[str, None]:
|
||||
return os.environ.pop(PATRONI_ENV_PREFIX + name.upper(), None)
|
||||
|
||||
for param in ('name', 'namespace', 'scope'):
|
||||
@@ -569,23 +381,6 @@ class Config(object):
|
||||
ret[param] = value
|
||||
|
||||
def _fix_log_env(name: str, oldname: str) -> None:
|
||||
"""Normalize a log related environment variable.
|
||||
|
||||
.. note::
|
||||
Patroni used to support different names for log related environment variables in the past. As the
|
||||
environment variables were renamed, this function takes care of mapping and normalizing the environment.
|
||||
|
||||
*name* is prefixed with :data:`~patroni.PATRONI_ENV_PREFIX` and ``LOG`` when searching in the
|
||||
environment.
|
||||
|
||||
*oldname* is prefixed with :data:`~patroni.PATRONI_ENV_PREFIX` when searching in the environment.
|
||||
|
||||
If both *name* and *oldname* are set in the environment, *name* takes precedence.
|
||||
|
||||
:param name: new name of a log related environment variable.
|
||||
:param oldname: original name of a log related environment variable.
|
||||
:type oldname: str
|
||||
"""
|
||||
value = _popenv(oldname)
|
||||
name = PATRONI_ENV_PREFIX + 'LOG_' + name.upper()
|
||||
if value and name not in os.environ:
|
||||
@@ -595,15 +390,6 @@ class Config(object):
|
||||
_fix_log_env(name, oldname)
|
||||
|
||||
def _set_section_values(section: str, params: List[str]) -> None:
|
||||
"""Get value of *params* environment variables that are related with *section*.
|
||||
|
||||
.. note::
|
||||
The values are retrieved from the environment and updated directly into the returning dictionary of
|
||||
:func:`_build_environment_configuration`.
|
||||
|
||||
:param section: configuration section the *params* belong to.
|
||||
:param params: name of the Patroni settings.
|
||||
"""
|
||||
for param in params:
|
||||
value = _popenv(section + '_' + param)
|
||||
if value:
|
||||
@@ -625,7 +411,6 @@ class Config(object):
|
||||
if value:
|
||||
ret['postgresql'].setdefault('bin_name', {})[binary] = value
|
||||
|
||||
# parse all values retrieved from the environment as Python objects, according to the expected type
|
||||
for first, second in (('restapi', 'allowlist_include_members'), ('ctl', 'insecure')):
|
||||
value = ret.get(first, {}).pop(second, None)
|
||||
if value:
|
||||
@@ -642,13 +427,7 @@ class Config(object):
|
||||
if value is not None:
|
||||
ret[first][second] = value
|
||||
|
||||
def _parse_list(value: str) -> Optional[List[str]]:
|
||||
"""Parse an YAML list *value* as a :class:`list`.
|
||||
|
||||
:param value: YAML list as a string.
|
||||
|
||||
:returns: *value* as :class:`list`.
|
||||
"""
|
||||
def _parse_list(value: str) -> Union[List[str], None]:
|
||||
if not (value.strip().startswith('-') or '[' in value):
|
||||
value = '[{0}]'.format(value)
|
||||
try:
|
||||
@@ -664,13 +443,7 @@ class Config(object):
|
||||
if value:
|
||||
ret[first][second] = value
|
||||
|
||||
def _parse_dict(value: str) -> Optional[Dict[str, Any]]:
|
||||
"""Parse an YAML dictionary *value* as a :class:`dict`.
|
||||
|
||||
:param value: YAML dictionary as a string.
|
||||
|
||||
:returns: *value* as :class:`dict`.
|
||||
"""
|
||||
def _parse_dict(value: str) -> Union[Dict[str, Any], None]:
|
||||
if not value.strip().startswith('{'):
|
||||
value = '{{{0}}}'.format(value)
|
||||
try:
|
||||
@@ -687,16 +460,9 @@ class Config(object):
|
||||
if value:
|
||||
ret[first][second] = value
|
||||
|
||||
def _get_auth(name: str, params: Collection[str] = _AUTH_ALLOWED_PARAMETERS[:2]) -> Dict[str, str]:
|
||||
"""Get authorization related environment variables *params* from section *name*.
|
||||
|
||||
:param name: name of a configuration section that may contain authorization *params*.
|
||||
:param params: the authorization settings that may be set under section *name*.
|
||||
|
||||
:returns: dictionary containing environment values for authorization *params* of section *name*.
|
||||
"""
|
||||
def _get_auth(name: str, params: Optional[Collection[str]] = None) -> Dict[str, str]:
|
||||
ret: Dict[str, str] = {}
|
||||
for param in params:
|
||||
for param in params or _AUTH_ALLOWED_PARAMETERS[:2]:
|
||||
value = _popenv(name + '_' + param)
|
||||
if value:
|
||||
ret[param] = value
|
||||
@@ -719,7 +485,7 @@ class Config(object):
|
||||
for param in list(os.environ.keys()):
|
||||
if param.startswith(PATRONI_ENV_PREFIX):
|
||||
# PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
|
||||
name, suffix = (param[len(PATRONI_ENV_PREFIX):].split('_', 1) + [''])[:2]
|
||||
name, suffix = (param[8:].split('_', 1) + [''])[:2]
|
||||
if suffix in ('HOST', 'HOSTS', 'PORT', 'USE_PROXIES', 'PROTOCOL', 'SRV', 'SRV_SUFFIX', 'URL', 'PROXY',
|
||||
'CACERT', 'CERT', 'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC', 'CONSISTENCY',
|
||||
'REGISTER_SERVICE', 'SERVICE_CHECK_INTERVAL', 'SERVICE_CHECK_TLS_SERVER_NAME',
|
||||
@@ -750,14 +516,14 @@ class Config(object):
|
||||
users = {}
|
||||
for param in list(os.environ.keys()):
|
||||
if param.startswith(PATRONI_ENV_PREFIX):
|
||||
name, suffix = (param[len(PATRONI_ENV_PREFIX):].rsplit('_', 1) + [''])[:2]
|
||||
name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
|
||||
# PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>
|
||||
# CREATE USER "<username>" WITH <OPTIONS> PASSWORD '<password>'
|
||||
if name and suffix == 'PASSWORD':
|
||||
password = os.environ.pop(param)
|
||||
if password:
|
||||
users[name] = {'password': password}
|
||||
options = os.environ.pop(param[:-9] + '_OPTIONS', None) # replace "_PASSWORD" with "_OPTIONS"
|
||||
options = os.environ.pop(param[:-9] + '_OPTIONS', None)
|
||||
options = options and _parse_list(options)
|
||||
if options:
|
||||
users[name]['options'] = options
|
||||
@@ -768,16 +534,6 @@ class Config(object):
|
||||
|
||||
def _build_effective_configuration(self, dynamic_configuration: Dict[str, Any],
|
||||
local_configuration: Dict[str, Union[Dict[str, Any], Any]]) -> Dict[str, Any]:
|
||||
"""Build effective configuration by merging *dynamic_configuration* and *local_configuration*.
|
||||
|
||||
.. note::
|
||||
*local_configuration* takes precedence over *dynamic_configuration* if a setting is defined in both.
|
||||
|
||||
:param dynamic_configuration: Patroni dynamic configuration.
|
||||
:param local_configuration: Patroni local configuration.
|
||||
|
||||
:returns: _description_
|
||||
"""
|
||||
config = self._safe_copy_dynamic_configuration(dynamic_configuration)
|
||||
for name, value in local_configuration.items():
|
||||
if name == 'citus': # remove invalid citus configuration
|
||||
@@ -841,57 +597,23 @@ class Config(object):
|
||||
return config
|
||||
|
||||
def get(self, key: str, default: Optional[Any] = None) -> Any:
|
||||
"""Get effective value of ``key`` setting from Patroni configuration root.
|
||||
|
||||
Designed to work the same way as :func:`dict.get`.
|
||||
|
||||
:param key: name of the setting.
|
||||
:param default: default value if *key* is not present in the effective configuration.
|
||||
|
||||
:returns: value of *key*, if present in the effective configuration, otherwise *default*.
|
||||
"""
|
||||
return self.__effective_configuration.get(key, default)
|
||||
|
||||
def __contains__(self, key: str) -> bool:
|
||||
"""Check if setting *key* is present in the effective configuration.
|
||||
|
||||
Designed to work the same way as :func:`dict.__contains__`.
|
||||
|
||||
:param key: name of the setting to be checked.
|
||||
|
||||
:returns: ``True`` if setting *key* exists in effective configuration, else ``False``.
|
||||
"""
|
||||
return key in self.__effective_configuration
|
||||
|
||||
def __getitem__(self, key: str) -> Any:
|
||||
"""Get value of setting *key* from effective configuration.
|
||||
|
||||
Designed to work the same way as :func:`dict.__getitem__`.
|
||||
|
||||
:param key: name of the setting.
|
||||
|
||||
:returns: value of setting *key*.
|
||||
|
||||
:raises:
|
||||
:class:`KeyError`: if *key* is not present in effective configuration.
|
||||
"""
|
||||
return self.__effective_configuration[key]
|
||||
|
||||
def copy(self) -> Dict[str, Any]:
|
||||
"""Get a deep copy of effective Patroni configuration.
|
||||
|
||||
:returns: a deep copy of the Patroni configuration.
|
||||
"""
|
||||
return deepcopy(self.__effective_configuration)
|
||||
|
||||
def get_global_config(self, cluster: Optional[Cluster]) -> GlobalConfig:
|
||||
def get_global_config(self, cluster: Union[Cluster, None]) -> GlobalConfig:
|
||||
"""Instantiate :class:`GlobalConfig` based on input.
|
||||
|
||||
Use the configuration from provided *cluster* (the most up-to-date) or from the
|
||||
local cache if *cluster.config* is not initialized or doesn't have a valid config.
|
||||
|
||||
:param cluster: the currently known cluster state from DCS.
|
||||
|
||||
:returns: :class:`GlobalConfig` object.
|
||||
:param cluster: the currently known cluster state from DCS
|
||||
:returns: :class:`GlobalConfig` object
|
||||
"""
|
||||
return get_global_config(cluster, self._dynamic_configuration)
|
||||
|
||||
@@ -1,463 +0,0 @@
|
||||
"""patroni ``--generate-config`` machinery."""
|
||||
import abc
|
||||
import logging
|
||||
import os
|
||||
import psutil
|
||||
import socket
|
||||
import sys
|
||||
import yaml
|
||||
|
||||
from getpass import getuser, getpass
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from psycopg import Cursor
|
||||
from psycopg2 import cursor
|
||||
|
||||
from . import psycopg
|
||||
from .config import Config
|
||||
from .exceptions import PatroniException
|
||||
from .postgresql.config import ConfigHandler, parse_dsn
|
||||
from .postgresql.misc import postgres_major_version_to_int
|
||||
from .utils import get_major_version, parse_bool, patch_config, read_stripped
|
||||
|
||||
|
||||
# Mapping between the libpq connection parameters and the environment variables.
|
||||
# This dict should be kept in sync with `patroni.utils._AUTH_ALLOWED_PARAMETERS`
|
||||
# (we use "username" in the Patroni config for some reason, other parameter names are the same).
|
||||
_AUTH_ALLOWED_PARAMETERS_MAPPING = {
|
||||
'user': 'PGUSER',
|
||||
'password': 'PGPASSWORD',
|
||||
'sslmode': 'PGSSLMODE',
|
||||
'sslcert': 'PGSSLCERT',
|
||||
'sslkey': 'PGSSLKEY',
|
||||
'sslpassword': '',
|
||||
'sslrootcert': 'PGSSLROOTCERT',
|
||||
'sslcrl': 'PGSSLCRL',
|
||||
'sslcrldir': 'PGSSLCRLDIR',
|
||||
'gssencmode': 'PGGSSENCMODE',
|
||||
'channel_binding': 'PGCHANNELBINDING'
|
||||
}
|
||||
_NO_VALUE_MSG = '#FIXME'
|
||||
|
||||
|
||||
def get_address() -> Tuple[str, str]:
|
||||
"""Try to get hostname and the ip address for it returned by :func:`~socket.gethostname`.
|
||||
|
||||
.. note::
|
||||
Can also return local ip.
|
||||
|
||||
:returns: tuple consisting of the hostname returned by :func:`~socket.gethostname`
|
||||
and the first element in the sorted list of the addresses returned by :func:`~socket.getaddrinfo`.
|
||||
Sorting guarantees it will prefer IPv4.
|
||||
If an exception occured, hostname and ip values are equal to :data:`~patroni.config_generator._NO_VALUE_MSG`.
|
||||
"""
|
||||
hostname = None
|
||||
try:
|
||||
hostname = socket.gethostname()
|
||||
return hostname, sorted(socket.getaddrinfo(hostname, 0, socket.AF_UNSPEC, socket.SOCK_STREAM, 0),
|
||||
key=lambda x: x[0])[0][4][0]
|
||||
except Exception as err:
|
||||
logging.warning('Failed to obtain address: %r', err)
|
||||
return _NO_VALUE_MSG, _NO_VALUE_MSG
|
||||
|
||||
|
||||
class AbstractConfigGenerator(abc.ABC):
|
||||
"""Object representing the generated Patroni config.
|
||||
|
||||
:ivar output_file: full path to the output file to be used.
|
||||
:ivar pg_major: integer representation of the major PostgreSQL version.
|
||||
:ivar config: dictionary used for the generated configuration storage.
|
||||
"""
|
||||
|
||||
_HOSTNAME, _IP = get_address()
|
||||
|
||||
def __init__(self, output_file: Optional[str]) -> None:
|
||||
"""Set up the output file (if passed), helper vars and the minimal config structure.
|
||||
|
||||
:param output_file: full path to the output file to be used.
|
||||
"""
|
||||
self.output_file = output_file
|
||||
self.pg_major = 0
|
||||
self.config = self.get_template_config()
|
||||
|
||||
self.generate()
|
||||
|
||||
@classmethod
|
||||
def get_template_config(cls) -> Dict[str, Any]:
|
||||
"""Generate a template config for further extension (e.g. in the inherited classes).
|
||||
|
||||
:returns: dictionary with the values gathered from Patroni env, hopefully defined hostname and ip address
|
||||
(otherwise set to :data:`~patroni.config_generator._NO_VALUE_MSG`), and some sane defaults.
|
||||
"""
|
||||
template_config: Dict[str, Any] = {
|
||||
'scope': _NO_VALUE_MSG,
|
||||
'name': cls._HOSTNAME,
|
||||
'postgresql': {
|
||||
'data_dir': _NO_VALUE_MSG,
|
||||
'connect_address': _NO_VALUE_MSG + ':5432',
|
||||
'listen': _NO_VALUE_MSG + ':5432',
|
||||
'bin_dir': '',
|
||||
'authentication': {
|
||||
'superuser': {
|
||||
'username': 'postgres',
|
||||
'password': _NO_VALUE_MSG
|
||||
},
|
||||
'replication': {
|
||||
'username': 'replicator',
|
||||
'password': _NO_VALUE_MSG
|
||||
}
|
||||
}
|
||||
},
|
||||
'restapi': {
|
||||
'connect_address': cls._IP + ':8008',
|
||||
'listen': cls._IP + ':8008'
|
||||
}
|
||||
}
|
||||
|
||||
dynamic_config = Config.get_default_config()
|
||||
# to properly dump CaseInsensitiveDict as YAML later
|
||||
dynamic_config['postgresql']['parameters'] = dict(dynamic_config['postgresql']['parameters'])
|
||||
config = Config('', None).local_configuration # Get values from env
|
||||
config.setdefault('bootstrap', {})['dcs'] = dynamic_config
|
||||
config.setdefault('postgresql', {})
|
||||
del config['bootstrap']['dcs']['standby_cluster']
|
||||
|
||||
patch_config(template_config, config)
|
||||
return template_config
|
||||
|
||||
@abc.abstractmethod
|
||||
def generate(self) -> None:
|
||||
"""Generate config and store in :attr:`~AbstractConfigGenerator.config`."""
|
||||
|
||||
def write_config(self) -> None:
|
||||
"""Write current :attr:`~AbstractConfigGenerator.config` to the output file if provided, to stdout otherwise."""
|
||||
if self.output_file:
|
||||
dir_path = os.path.dirname(self.output_file)
|
||||
if dir_path and not os.path.isdir(dir_path):
|
||||
os.makedirs(dir_path)
|
||||
with open(self.output_file, 'w', encoding='UTF-8') as output_file:
|
||||
yaml.safe_dump(self.config, output_file, default_flow_style=False, allow_unicode=True)
|
||||
else:
|
||||
yaml.safe_dump(self.config, sys.stdout, default_flow_style=False, allow_unicode=True)
|
||||
|
||||
|
||||
class SampleConfigGenerator(AbstractConfigGenerator):
|
||||
"""Object representing the generated sample Patroni config.
|
||||
|
||||
Sane defults are used based on the gathered PG version.
|
||||
"""
|
||||
|
||||
@property
|
||||
def get_auth_method(self) -> str:
|
||||
"""Return the preferred authentication method for a specific PG version if provided or the default ``md5``.
|
||||
|
||||
:returns: :class:`str` value for the preferred authentication method.
|
||||
"""
|
||||
return 'scram-sha-256' if self.pg_major and self.pg_major >= 100000 else 'md5'
|
||||
|
||||
def _get_int_major_version(self) -> int:
|
||||
"""Get major PostgreSQL version from the binary as an integer.
|
||||
|
||||
:returns: an integer PostgreSQL major version representation gathered from the PostgreSQL binary.
|
||||
See :func:`~patroni.postgresql.misc.postgres_major_version_to_int` and
|
||||
:func:`~patroni.utils.get_major_version`.
|
||||
"""
|
||||
postgres_bin = ((self.config.get('postgresql') or {}).get('bin_name') or {}).get('postgres', 'postgres')
|
||||
return postgres_major_version_to_int(get_major_version(self.config['postgresql'].get('bin_dir'), postgres_bin))
|
||||
|
||||
def generate(self) -> None:
|
||||
"""Generate sample config using some sane defaults and update :attr:`~AbstractConfigGenerator.config`."""
|
||||
self.pg_major = self._get_int_major_version()
|
||||
|
||||
self.config['postgresql']['parameters'] = {'password_encryption': self.get_auth_method}
|
||||
username = self.config["postgresql"]["authentication"]["replication"]["username"]
|
||||
self.config['postgresql']['pg_hba'] = [
|
||||
f'host all all all {self.get_auth_method}',
|
||||
f'host replication {username} all {self.get_auth_method}'
|
||||
]
|
||||
|
||||
# add version-specific configuration
|
||||
wal_keep_param = 'wal_keep_segments' if self.pg_major < 130000 else 'wal_keep_size'
|
||||
self.config['bootstrap']['dcs']['postgresql']['parameters'][wal_keep_param] = \
|
||||
ConfigHandler.CMDLINE_OPTIONS[wal_keep_param][0]
|
||||
|
||||
self.config['bootstrap']['dcs']['postgresql']['use_pg_rewind'] = True
|
||||
if self.pg_major >= 110000:
|
||||
self.config['postgresql']['authentication'].setdefault(
|
||||
'rewind', {'username': 'rewind_user'}).setdefault('password', _NO_VALUE_MSG)
|
||||
|
||||
|
||||
class RunningClusterConfigGenerator(AbstractConfigGenerator):
|
||||
"""Object representing the Patroni config generated using information gathered from the running instance.
|
||||
|
||||
:ivar dsn: DSN string for the local instance to get GUC values from (if provided).
|
||||
:ivar parsed_dsn: DSN string parsed into a dictionary (see :func:`~patroni.postgresql.config.parse_dsn`).
|
||||
"""
|
||||
|
||||
def __init__(self, output_file: Optional[str] = None, dsn: Optional[str] = None) -> None:
|
||||
"""Additionally store the passed dsn (if any) in both original and parsed version and run config generation.
|
||||
|
||||
:param output_file: full path to the output file to be used.
|
||||
:param dsn: DSN string for the local instance to get GUC values from.
|
||||
|
||||
:raises:
|
||||
:exc:`~patroni.exceptions.PatroniException`: if DSN parsing failed.
|
||||
"""
|
||||
self.dsn = dsn
|
||||
self.parsed_dsn = {}
|
||||
|
||||
super().__init__(output_file)
|
||||
|
||||
@property
|
||||
def _get_hba_conn_types(self) -> Tuple[str, ...]:
|
||||
"""Return the connection types allowed.
|
||||
|
||||
If :attr:`~RunningClusterConfigGenerator.pg_major` is defined, adds additional parameters
|
||||
for PostgreSQL version >=16.
|
||||
|
||||
:returns: tuple of the connection methods allowed.
|
||||
"""
|
||||
allowed_types = ('local', 'host', 'hostssl', 'hostnossl', 'hostgssenc', 'hostnogssenc')
|
||||
if self.pg_major and self.pg_major >= 160000:
|
||||
allowed_types += ('include', 'include_if_exists', 'include_dir')
|
||||
return allowed_types
|
||||
|
||||
@property
|
||||
def _required_pg_params(self) -> List[str]:
|
||||
"""PG configuration prameters that have to be always present in the generated config.
|
||||
|
||||
:returns: list of the parameter names.
|
||||
"""
|
||||
return ['hba_file', 'ident_file', 'config_file', 'data_directory'] + \
|
||||
list(ConfigHandler.CMDLINE_OPTIONS.keys())
|
||||
|
||||
def _get_bin_dir_from_running_instance(self) -> str:
|
||||
"""Define the directory postgres binaries reside using postmaster's pid executable.
|
||||
|
||||
:returns: path to the PostgreSQL binaries directory.
|
||||
|
||||
:raises:
|
||||
:exc:`~patroni.exceptions.PatroniException`: if:
|
||||
|
||||
* pid could not be obtained from the ``postmaster.pid`` file; or
|
||||
* :exc:`OSError` occured during ``postmaster.pid`` file handling; or
|
||||
* the obtained postmaster pid doesn't exist.
|
||||
"""
|
||||
postmaster_pid = None
|
||||
data_dir = self.config['postgresql']['data_dir']
|
||||
try:
|
||||
with open(f"{data_dir}/postmaster.pid", 'r') as pid_file:
|
||||
postmaster_pid = pid_file.readline()
|
||||
if not postmaster_pid:
|
||||
raise PatroniException('Failed to obtain postmaster pid from postmaster.pid file')
|
||||
postmaster_pid = int(postmaster_pid.strip())
|
||||
except OSError as err:
|
||||
raise PatroniException(f'Error while reading postmaster.pid file: {err}')
|
||||
try:
|
||||
return os.path.dirname(psutil.Process(postmaster_pid).exe())
|
||||
except psutil.NoSuchProcess:
|
||||
raise PatroniException("Obtained postmaster pid doesn't exist.")
|
||||
|
||||
@contextmanager
|
||||
def _get_connection_cursor(self) -> Iterator[Union['cursor', 'Cursor[Any]']]:
|
||||
"""Get cursor for the PG connection established based on the stored information.
|
||||
|
||||
:raises:
|
||||
:exc:`~patroni.exceptions.PatroniException`: if :exc:`psycopg.Error` occured.
|
||||
"""
|
||||
try:
|
||||
conn = psycopg.connect(dsn=self.dsn,
|
||||
password=self.config['postgresql']['authentication']['superuser']['password'])
|
||||
with conn.cursor() as cur:
|
||||
yield cur
|
||||
conn.close()
|
||||
except psycopg.Error as e:
|
||||
raise PatroniException(f'Failed to establish PostgreSQL connection: {e}')
|
||||
|
||||
def _set_pg_params(self, cur: Union['cursor', 'Cursor[Any]']) -> None:
|
||||
"""Extend :attr:`~RunningClusterConfigGenerator.config` with the actual PG GUCs values.
|
||||
|
||||
THe following GUC values are set:
|
||||
|
||||
* Non-internal having configuration file, postmaster command line or environment variable
|
||||
as a source.
|
||||
|
||||
* List of the always required parameters (see :meth:`~RunningClusterConfigGenerator._required_pg_params`).
|
||||
|
||||
:param cur: connection cursor to use.
|
||||
"""
|
||||
cur.execute("SELECT name, current_setting(name) FROM pg_settings "
|
||||
"WHERE context <> 'internal' "
|
||||
"AND source IN ('configuration file', 'command line', 'environment variable') "
|
||||
"AND category <> 'Write-Ahead Log / Recovery Target' "
|
||||
"AND setting <> '(disabled)' "
|
||||
"OR name = ANY(%s)", (self._required_pg_params,))
|
||||
|
||||
helper_dict = dict.fromkeys(['port', 'listen_addresses'])
|
||||
self.config['postgresql'].setdefault('parameters', {})
|
||||
for param, value in cur.fetchall():
|
||||
if param == 'data_directory':
|
||||
self.config['postgresql']['data_dir'] = value
|
||||
elif param == 'cluster_name' and value:
|
||||
self.config['scope'] = value
|
||||
elif param in ('archive_command', 'restore_command',
|
||||
'archive_cleanup_command', 'recovery_end_command',
|
||||
'ssl_passphrase_command', 'hba_file',
|
||||
'ident_file', 'config_file'):
|
||||
# write commands to the local config due to security implications
|
||||
# write hba/ident/config_file to local config to ensure they are not removed later
|
||||
self.config['postgresql']['parameters'][param] = value
|
||||
elif param in helper_dict:
|
||||
helper_dict[param] = value
|
||||
else:
|
||||
self.config['bootstrap']['dcs']['postgresql']['parameters'][param] = value
|
||||
|
||||
connect_port = self.parsed_dsn.get('port', os.getenv('PGPORT', helper_dict['port']))
|
||||
self.config['postgresql']['connect_address'] = f'{self._IP}:{connect_port}'
|
||||
self.config['postgresql']['listen'] = f'{helper_dict["listen_addresses"]}:{helper_dict["port"]}'
|
||||
|
||||
def _set_su_params(self) -> None:
|
||||
"""Extend :attr:`~RunningClusterConfigGenerator.config` with the superuser auth information.
|
||||
|
||||
Information set is based on the options used for connection.
|
||||
"""
|
||||
su_params: Dict[str, str] = {}
|
||||
for conn_param, env_var in _AUTH_ALLOWED_PARAMETERS_MAPPING.items():
|
||||
val = self.parsed_dsn.get(conn_param, os.getenv(env_var))
|
||||
if val:
|
||||
su_params[conn_param] = val
|
||||
patroni_env_su_username = ((self.config.get('authentication') or {}).get('superuser') or {}).get('username')
|
||||
patroni_env_su_pwd = ((self.config.get('authentication') or {}).get('superuser') or {}).get('password')
|
||||
# because we use "username" in the config for some reason
|
||||
su_params['username'] = su_params.pop('user', patroni_env_su_username) or getuser()
|
||||
su_params['password'] = su_params.get('password', patroni_env_su_pwd) or \
|
||||
getpass('Please enter the user password:')
|
||||
self.config['postgresql']['authentication'] = {
|
||||
'superuser': su_params,
|
||||
'replication': {'username': _NO_VALUE_MSG, 'password': _NO_VALUE_MSG}
|
||||
}
|
||||
|
||||
def _set_conf_files(self) -> None:
|
||||
"""Extend :attr:`~RunningClusterConfigGenerator.config` with ``pg_hba.conf`` and ``pg_ident.conf`` content.
|
||||
|
||||
.. note::
|
||||
This function only defines ``postgresql.pg_hba`` and ``postgresql.pg_ident`` when
|
||||
``hba_file`` and ``ident_file`` are set to the defaults. It may happen these files
|
||||
are located outside of ``PGDATA`` and Patroni doesn't have write permissions for them.
|
||||
|
||||
:raises:
|
||||
:exc:`~patroni.exceptions.PatroniException`: if :exc:`OSError` occured during the conf files handling.
|
||||
"""
|
||||
default_hba_path = os.path.join(self.config['postgresql']['data_dir'], 'pg_hba.conf')
|
||||
if self.config['postgresql']['parameters']['hba_file'] == default_hba_path:
|
||||
try:
|
||||
self.config['postgresql']['pg_hba'] = list(
|
||||
filter(lambda i: i and i.split()[0] in self._get_hba_conn_types, read_stripped(default_hba_path)))
|
||||
except OSError as err:
|
||||
raise PatroniException(f'Failed to read pg_hba.conf: {err}')
|
||||
|
||||
default_ident_path = os.path.join(self.config['postgresql']['data_dir'], 'pg_ident.conf')
|
||||
if self.config['postgresql']['parameters']['ident_file'] == default_ident_path:
|
||||
try:
|
||||
self.config['postgresql']['pg_ident'] = [i for i in read_stripped(default_ident_path)
|
||||
if i and not i.startswith('#')]
|
||||
except OSError as err:
|
||||
raise PatroniException(f'Failed to read pg_ident.conf: {err}')
|
||||
if not self.config['postgresql']['pg_ident']:
|
||||
del self.config['postgresql']['pg_ident']
|
||||
|
||||
def _enrich_config_from_running_instance(self) -> None:
|
||||
"""Extend :attr:`~RunningClusterConfigGenerator.config` with the values gathered from the running instance.
|
||||
|
||||
Retrieve the following information from the running PostgreSQL instance:
|
||||
|
||||
* superuser auth parameters (see :meth:`~RunningClusterConfigGenerator._set_su_params`);
|
||||
* some GUC values (see :meth:`~RunningClusterConfigGenerator._set_pg_params`);
|
||||
* ``postgresql.connect_address``, ``postgresql.listen``;
|
||||
* ``postgresql.pg_hba`` and ``postgresql.pg_ident`` (see :meth:`~RunningClusterConfigGenerator._set_conf_files`)
|
||||
|
||||
And redefine ``scope`` with the ``cluster_name`` GUC value if set.
|
||||
|
||||
:raises:
|
||||
:exc:`~patroni.exceptions.PatroniException`: if the provided user doesn't have superuser privileges.
|
||||
"""
|
||||
self._set_su_params()
|
||||
|
||||
with self._get_connection_cursor() as cur:
|
||||
self.pg_major = getattr(cur.connection, 'server_version', 0)
|
||||
|
||||
if not parse_bool(cur.connection.info.parameter_status('is_superuser')):
|
||||
raise PatroniException('The provided user does not have superuser privilege')
|
||||
|
||||
self._set_pg_params(cur)
|
||||
|
||||
self._set_conf_files()
|
||||
|
||||
def generate(self) -> None:
|
||||
"""Generate config using the info gathered from the specified running PG instance.
|
||||
|
||||
Result is written to :attr:`~RunningClusterConfigGenerator.config`.
|
||||
"""
|
||||
if self.dsn:
|
||||
self.parsed_dsn = parse_dsn(self.dsn) or {}
|
||||
if not self.parsed_dsn:
|
||||
raise PatroniException('Failed to parse DSN string')
|
||||
|
||||
self._enrich_config_from_running_instance()
|
||||
self.config['postgresql']['bin_dir'] = self._get_bin_dir_from_running_instance()
|
||||
|
||||
|
||||
def generate_config(output_file: str, sample: bool, dsn: Optional[str]) -> None:
|
||||
"""Generate Patroni configuration file.
|
||||
|
||||
Gather all the available non-internal GUC values having configuration file, postmaster command line or environment
|
||||
variable as a source and store them in the appropriate part of Patroni configuration (``postgresql.parameters`` or
|
||||
``bootstrap.dcs.postgresql.parameters``). Either the provided DSN (takes precedence) or PG ENV vars will be used
|
||||
for the connection. If password is not provided, it should be entered via prompt.
|
||||
|
||||
The created configuration contains:
|
||||
* ``scope``: ``cluster_name`` GUC value or ``PATRONI_SCOPE ENV`` variable value if available.
|
||||
* ``name``: ``PATRONI_NAME`` ENV variable value if set, otherwise hostname.
|
||||
|
||||
* ``bootstrap.dcs``: section with all the parameters (incl. the majority of PG GUCs) set to their default values
|
||||
defined by Patroni and adjusted by the source instances's configuration values.
|
||||
|
||||
* ``postgresql.parameters``: the source instance's ``archive_command``, ``restore_command``,
|
||||
``archive_cleanup_command``, ``recovery_end_command``, ``ssl_passphrase_command``, ``hba_file``, ``ident_file``,
|
||||
``config_file`` GUC values.
|
||||
|
||||
* ``postgresql.bin_dir``: path to Postgres binaries gathered from the running instance or, if not available,
|
||||
the value of ``PATRONI_POSTGRESQL_BIN_DIR`` ENV variable. Otherwise, an empty string.
|
||||
|
||||
* ``postgresql.datadir``: the value gathered from the corresponding PG GUC.
|
||||
* ``postgresql.listen``: source instance's ``listen_addresses`` and port GUC values.
|
||||
* ``postgresql.connect_address``: if possible, generated from the connection params.
|
||||
* ``postgresql.authentication``:
|
||||
|
||||
* superuser and replication users defined (if possible, usernames are set from the respective Patroni ENV vars,
|
||||
otherwise the default ``postgres`` and ``replicator`` values are used).
|
||||
If not a sample config, either DSN or PG ENV vars are used to define superuser authentication parameters.
|
||||
|
||||
* rewind user is defined only for sample config, if PG version can be defined and PG version is >=11
|
||||
(if possible, username is set from the respective Patroni ENV var).
|
||||
|
||||
* ``bootstrap.dcs.postgresql.use_pg_rewind`` set to ``True`` for a sample config only.
|
||||
* ``postgresql.pg_hba`` defaults or the lines gathered from the source instance's ``hba_file``.
|
||||
* ``postgresql.pg_ident`` the lines gathered from the source instance's ``ident_file``.
|
||||
|
||||
:param output_file: Full path to the configuration file to be used. If not provided, result is sent to ``stdout``.
|
||||
:param sample: Optional flag. If set, no source instance will be used - generate config with some sane defaults.
|
||||
:param dsn: Optional DSN string for the local instance to get GUC values from.
|
||||
"""
|
||||
try:
|
||||
if sample:
|
||||
config_generator = SampleConfigGenerator(output_file)
|
||||
else:
|
||||
config_generator = RunningClusterConfigGenerator(output_file, dsn)
|
||||
|
||||
config_generator.write_config()
|
||||
except PatroniException as e:
|
||||
sys.exit(str(e))
|
||||
except Exception as e:
|
||||
sys.exit(f'Unexpected exception: {e}')
|
||||
+107
-86
@@ -16,6 +16,8 @@ import click
|
||||
import codecs
|
||||
import copy
|
||||
import datetime
|
||||
import dateutil.parser
|
||||
import dateutil.tz
|
||||
import difflib
|
||||
import io
|
||||
import json
|
||||
@@ -44,12 +46,10 @@ try:
|
||||
except ImportError: # pragma: no cover
|
||||
from cdiff import markup_to_pager, PatchStream # pyright: ignore [reportMissingModuleSource]
|
||||
|
||||
from .config import Config, get_global_config
|
||||
from .dcs import get_dcs as _get_dcs, AbstractDCS, Cluster, Member
|
||||
from .exceptions import PatroniException
|
||||
from .manual_failover import ManualFailover
|
||||
from .postgresql.misc import postgres_version_to_int
|
||||
from .utils import cluster_as_json, parse_schedule, patch_config, polling_loop
|
||||
from .utils import cluster_as_json, patch_config, polling_loop
|
||||
from .request import PatroniRequest
|
||||
from .version import __version__
|
||||
|
||||
@@ -225,6 +225,8 @@ def load_config(path: str, dcs_url: Optional[str]) -> Dict[str, Any]:
|
||||
:raises:
|
||||
:class:`PatroniCtlException`: if *path* does not exist or is not readable.
|
||||
"""
|
||||
from patroni.config import Config
|
||||
|
||||
if not (os.path.exists(path) and os.access(path, os.R_OK)):
|
||||
if path != CONFIG_FILE_PATH: # bail if non-default config location specified but file not found / readable
|
||||
raise PatroniCtlException('Provided config file {0} not existing or no read rights.'
|
||||
@@ -560,9 +562,10 @@ def get_cursor(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], conn
|
||||
from . import psycopg
|
||||
conn = psycopg.connect(**params)
|
||||
cursor = conn.cursor()
|
||||
# If we want ``any`` node we are fine to return the cursor
|
||||
# If we want ``any`` node we are fine to return the cursor. ``None`` is similar to ``any`` at this point, as it's
|
||||
# been dealt with through :func:`get_any_member`.
|
||||
# If we want the Patroni leader node, :func:`get_any_member` already checks that for us
|
||||
if role in ('any', 'leader'):
|
||||
if role in (None, 'any', 'leader'):
|
||||
return cursor
|
||||
|
||||
# If we want something other than ``any`` or ``leader``, then we do not rely only on the DCS information about
|
||||
@@ -644,8 +647,7 @@ def get_members(obj: Dict[str, Any], cluster: Cluster, cluster_name: str, member
|
||||
if member_names:
|
||||
member_names = list(set(member_names) & candidates)
|
||||
if not member_names:
|
||||
raise PatroniCtlException(
|
||||
'No{0} among provided members'.format('t a single cluster member' if role == 'any' else ' ' + role))
|
||||
raise PatroniCtlException('No {0} among provided members'.format(role))
|
||||
elif action != 'reinitialize':
|
||||
member_names = list(candidates)
|
||||
|
||||
@@ -857,9 +859,11 @@ def query_member(obj: Dict[str, Any], cluster: Cluster, group: Optional[int],
|
||||
|
||||
if cursor is None:
|
||||
if member is not None:
|
||||
message = 'No connection to member {0} is available'.format(member)
|
||||
message = f'No connection to member {member} is available'
|
||||
elif role is not None:
|
||||
message = f'No connection to role {role} is available'
|
||||
else:
|
||||
message = 'No connection to role={0} is available'.format(role)
|
||||
message = 'No connection is available'
|
||||
logging.debug(message)
|
||||
return [[timestamp(0), message]], None
|
||||
|
||||
@@ -945,6 +949,43 @@ def check_response(response: urllib3.response.HTTPResponse, member_name: str,
|
||||
return True
|
||||
|
||||
|
||||
def parse_scheduled(scheduled: Optional[str]) -> Optional[datetime.datetime]:
|
||||
"""Parse a string *scheduled* timestamp as a :class:`~datetime.datetime` object.
|
||||
|
||||
:param scheduled: string representation of the timestamp. May also be ``now``.
|
||||
|
||||
:returns: the corresponding :class:`~datetime.datetime` object, if *scheduled* is not ``now``, otherwise ``None``.
|
||||
|
||||
:raises:
|
||||
:class:`PatroniCtlException`: if unable to parse *scheduled* from :class:`str` to :class:`~datetime.datetime`.
|
||||
|
||||
:Example:
|
||||
|
||||
>>> parse_scheduled(None) is None
|
||||
True
|
||||
|
||||
>>> parse_scheduled('now') is None
|
||||
True
|
||||
|
||||
>>> parse_scheduled('2023-05-29T04:32:31')
|
||||
datetime.datetime(2023, 5, 29, 4, 32, 31, tzinfo=tzlocal())
|
||||
|
||||
>>> parse_scheduled('2023-05-29T04:32:31-3')
|
||||
datetime.datetime(2023, 5, 29, 4, 32, 31, tzinfo=tzoffset(None, -10800))
|
||||
"""
|
||||
if scheduled is not None and (scheduled or 'now') != 'now':
|
||||
try:
|
||||
scheduled_at = dateutil.parser.parse(scheduled)
|
||||
if scheduled_at.tzinfo is None:
|
||||
scheduled_at = scheduled_at.replace(tzinfo=dateutil.tz.tzlocal())
|
||||
except (ValueError, TypeError):
|
||||
message = 'Unable to parse scheduled timestamp ({0}). It should be in an unambiguous format (e.g. ISO 8601)'
|
||||
raise PatroniCtlException(message.format(scheduled))
|
||||
return scheduled_at
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@ctl.command('reload', help='Reload cluster member configuration')
|
||||
@click.argument('cluster_name')
|
||||
@click.argument('member_names', nargs=-1)
|
||||
@@ -975,6 +1016,7 @@ def reload(obj: Dict[str, Any], cluster_name: str, member_names: List[str],
|
||||
if r.status == 200:
|
||||
click.echo('No changes to apply on member {0}'.format(member.name))
|
||||
elif r.status == 202:
|
||||
from patroni.config import get_global_config
|
||||
config = get_global_config(cluster)
|
||||
click.echo('Reload request received for member {0} and will be processed within {1} seconds'.format(
|
||||
member.name, config.get('loop_wait') or dcs.loop_wait)
|
||||
@@ -1024,20 +1066,16 @@ def restart(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member
|
||||
* *version* could not be parsed; or
|
||||
* a restart is attempted against a cluster that is in maintenance mode.
|
||||
"""
|
||||
action = 'restart'
|
||||
cluster = get_dcs(obj, cluster_name, group).get_cluster()
|
||||
|
||||
members = get_members(obj, cluster, cluster_name, member_names, role, force, action, False, group=group)
|
||||
members = get_members(obj, cluster, cluster_name, member_names, role, force, 'restart', False, group=group)
|
||||
if scheduled is None and not force:
|
||||
next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M+00')
|
||||
next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M')
|
||||
scheduled = click.prompt('When should the restart take place (e.g. ' + next_hour + ') ',
|
||||
type=str, default='now')
|
||||
scheduled = scheduled if scheduled != 'now' else None
|
||||
|
||||
parse_result, scheduled_at = parse_schedule(scheduled)
|
||||
if parse_result:
|
||||
raise PatroniCtlException(parse_result.value[0].format(action=action))
|
||||
confirm_members_action(members, force, action, scheduled_at)
|
||||
scheduled_at = parse_scheduled(scheduled)
|
||||
confirm_members_action(members, force, 'restart', scheduled_at)
|
||||
|
||||
if p_any:
|
||||
random.shuffle(members)
|
||||
@@ -1060,6 +1098,7 @@ def restart(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member
|
||||
content['postgres_version'] = version
|
||||
|
||||
if scheduled_at:
|
||||
from patroni.config import get_global_config
|
||||
if get_global_config(cluster).is_paused:
|
||||
raise PatroniCtlException("Can't schedule restart in the paused state")
|
||||
content['schedule'] = scheduled_at.isoformat()
|
||||
@@ -1179,9 +1218,6 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s
|
||||
click.echo('Current cluster topology')
|
||||
output_members(obj, cluster, cluster_name, group=group)
|
||||
|
||||
# Define everything missing via interactive input or available cluster info (if force mode)
|
||||
|
||||
# Require Citus group
|
||||
if obj.get('citus') and group is None:
|
||||
if force:
|
||||
raise PatroniCtlException('For Citus clusters the --group must me specified')
|
||||
@@ -1190,82 +1226,72 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s
|
||||
dcs = get_dcs(obj, cluster_name, group)
|
||||
cluster = dcs.get_cluster()
|
||||
|
||||
global_config = get_global_config(cluster)
|
||||
if action == 'switchover' and (cluster.leader is None or not cluster.leader.name):
|
||||
raise PatroniCtlException('This cluster has no leader')
|
||||
|
||||
# Leader is required for switchover only
|
||||
if action == 'switchover' and leader is None:
|
||||
if cluster.leader is None or not cluster.leader.name:
|
||||
raise PatroniCtlException('This cluster has no leader')
|
||||
if force:
|
||||
leader = cluster.leader.name
|
||||
if leader is None:
|
||||
if force or action == 'failover':
|
||||
leader = cluster.leader and cluster.leader.name
|
||||
else:
|
||||
prompt = 'Standby Leader' if global_config.is_standby_cluster else 'Primary'
|
||||
leader = click.prompt(prompt, type=str, default=(cluster.leader and cluster.leader.name))
|
||||
from patroni.config import get_global_config
|
||||
prompt = 'Standby Leader' if get_global_config(cluster).is_standby_cluster else 'Primary'
|
||||
leader = click.prompt(prompt, type=str, default=(cluster.leader and cluster.leader.member.name))
|
||||
|
||||
if leader is not None and cluster.leader and cluster.leader.member.name != leader:
|
||||
raise PatroniCtlException('Member {0} is not the leader of cluster {1}'.format(leader, cluster_name))
|
||||
|
||||
# excluding members with nofailover tag
|
||||
candidate_names = [str(m.name) for m in cluster.members if m.name != leader and not m.nofailover]
|
||||
# We sort the names for consistent output to the client
|
||||
candidate_names.sort()
|
||||
|
||||
if not candidate_names:
|
||||
raise PatroniCtlException('No candidates found to {0} to'.format(action))
|
||||
|
||||
if candidate is None and not force:
|
||||
# Check if there are any candidates available at all
|
||||
candidate_names = [str(m.name) for m in cluster.members if m.name != leader and not m.nofailover]
|
||||
if not candidate_names:
|
||||
raise PatroniCtlException('No candidates found to {0} to'.format(action))
|
||||
candidate_names.sort() # we sort the names for consistent output to the client
|
||||
candidate = click.prompt('Candidate ' + str(candidate_names), type=str, default='')
|
||||
|
||||
# We allow manual failover to an aync node in the sync mode, so we better ask for the confirmation
|
||||
if all((not force,
|
||||
action == 'failover',
|
||||
global_config.is_synchronous_mode,
|
||||
not cluster.sync.is_empty,
|
||||
not cluster.sync.matches(candidate, True))):
|
||||
if click.confirm(f'Are you sure you want to failover to the asynchronous node {candidate}'):
|
||||
raise PatroniCtlException('Aborting ' + action)
|
||||
if action == 'failover' and not candidate:
|
||||
raise PatroniCtlException('Failover could be performed only to a specific candidate')
|
||||
|
||||
if action == 'switchover' and scheduled is None and not force:
|
||||
next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M+00')
|
||||
scheduled = click.prompt('When should the switchover take place (e.g. ' + next_hour + ') ',
|
||||
type=str, default='now')
|
||||
scheduled = scheduled if scheduled != 'now' else None
|
||||
if candidate == leader:
|
||||
raise PatroniCtlException(action.title() + ' target and source are the same.')
|
||||
|
||||
# Now, when we collected all the possible info, run checks
|
||||
manual_failover = ManualFailover(action, cluster, leader, candidate, scheduled,
|
||||
global_config.is_paused, global_config.is_synchronous_mode)
|
||||
|
||||
result_text, _ = manual_failover.run_precheck().value
|
||||
if result_text:
|
||||
raise PatroniCtlException(result_text.format(action=action, leader=leader, candidate=candidate,
|
||||
cluster_name=cluster_name))
|
||||
if candidate and candidate not in candidate_names:
|
||||
raise PatroniCtlException('Member {0} does not exist in cluster {1}'.format(candidate, cluster_name))
|
||||
|
||||
scheduled_at_str = None
|
||||
scheduled_at = None
|
||||
|
||||
if action == 'switchover':
|
||||
parse_result, scheduled_at = manual_failover.parse_scheduled()
|
||||
if parse_result:
|
||||
raise PatroniCtlException(parse_result.value[0].format(action=action))
|
||||
if scheduled is None and not force:
|
||||
next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M')
|
||||
scheduled = click.prompt('When should the switchover take place (e.g. ' + next_hour + ' ) ',
|
||||
type=str, default='now')
|
||||
|
||||
scheduled_at = parse_scheduled(scheduled)
|
||||
if scheduled_at:
|
||||
from patroni.config import get_global_config
|
||||
if get_global_config(cluster).is_paused:
|
||||
raise PatroniCtlException("Can't schedule switchover in the paused state")
|
||||
scheduled_at_str = scheduled_at.isoformat()
|
||||
|
||||
# By now we have established that the leader exists and the candidate exists,
|
||||
# so confirm the action that is about to be run
|
||||
if not force:
|
||||
demote_msg = f', demoting current leader {cluster.leader.name}' if cluster.leader else ''
|
||||
if scheduled_at_str:
|
||||
# only switchover can be scheduled
|
||||
if not click.confirm(f'Are you sure you want to schedule a switchover in the cluster '
|
||||
f'{cluster_name} at {scheduled_at_str}{demote_msg}?'):
|
||||
# action as a var to catch a regression in the tests
|
||||
raise PatroniCtlException('Aborting scheduled ' + action)
|
||||
else:
|
||||
if not click.confirm(f'Are you sure you want to perform a {action} in the cluster {cluster_name}{demote_msg}?'):
|
||||
raise PatroniCtlException('Aborting ' + action)
|
||||
|
||||
# And finally the actual work
|
||||
failover_value = {'candidate': candidate}
|
||||
if action == 'switchover':
|
||||
failover_value['leader'] = leader
|
||||
if scheduled_at_str:
|
||||
failover_value['scheduled_at'] = scheduled_at_str
|
||||
failover_value = {'leader': leader, 'candidate': candidate, 'scheduled_at': scheduled_at_str}
|
||||
|
||||
logging.debug(failover_value)
|
||||
|
||||
# By now we have established that the leader exists and the candidate exists
|
||||
if not force:
|
||||
demote_msg = ', demoting current leader ' + leader if leader else ''
|
||||
if scheduled_at_str:
|
||||
if not click.confirm('Are you sure you want to schedule {0} of cluster {1} at {2}{3}?'
|
||||
.format(action, cluster_name, scheduled_at_str, demote_msg)):
|
||||
raise PatroniCtlException('Aborting scheduled ' + action)
|
||||
else:
|
||||
if not click.confirm('Are you sure you want to {0} cluster {1}{2}?'
|
||||
.format(action, cluster_name, demote_msg)):
|
||||
raise PatroniCtlException('Aborting ' + action)
|
||||
|
||||
r = None
|
||||
try:
|
||||
member = cluster.leader.member if cluster.leader else candidate and cluster.get_member(candidate, False)
|
||||
@@ -1309,8 +1335,6 @@ def failover(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
|
||||
|
||||
.. note::
|
||||
If *leader* is given perform a switchover instead of a failover.
|
||||
This behavior is deprecated. ``--leader`` option support will be
|
||||
removed in the next major release.
|
||||
|
||||
.. seealso::
|
||||
Refer to :func:`_do_failover_or_switchover` for details.
|
||||
@@ -1324,12 +1348,7 @@ def failover(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
|
||||
:param candidate: name of a standby member to be promoted. Nodes that are tagged with ``nofailover`` cannot be used.
|
||||
:param force: perform the failover or switchover without asking for confirmations.
|
||||
"""
|
||||
action = 'failover'
|
||||
if leader:
|
||||
action = 'switchover'
|
||||
click.echo(click.style(
|
||||
'Supplying a leader name using this command is deprecated and will be removed in a future version of'
|
||||
' Patroni, change your scripts to use `switchover` instead.\nExecuting switchover!', fg='red'))
|
||||
action = 'switchover' if leader else 'failover'
|
||||
_do_failover_or_switchover(obj, action, cluster_name, group, leader, candidate, force)
|
||||
|
||||
|
||||
@@ -1702,6 +1721,7 @@ def wait_until_pause_is_applied(dcs: AbstractDCS, paused: bool, old_cluster: Clu
|
||||
:param old_cluster: original cluster information before pause or unpause has been requested. Used to report which
|
||||
nodes are still pending to have ``pause`` equal *paused* at a given point in time.
|
||||
"""
|
||||
from patroni.config import get_global_config
|
||||
config = get_global_config(old_cluster)
|
||||
|
||||
click.echo("'{0}' request sent, waiting until it is recognized by all nodes".format(paused and 'pause' or 'resume'))
|
||||
@@ -1739,6 +1759,7 @@ def toggle_pause(config: Dict[str, Any], cluster_name: str, group: Optional[int]
|
||||
* ``pause`` state is already *paused*; or
|
||||
* cluster contains no accessible members.
|
||||
"""
|
||||
from patroni.config import get_global_config
|
||||
dcs = get_dcs(config, cluster_name, group)
|
||||
cluster = dcs.get_cluster()
|
||||
if get_global_config(cluster).is_paused == paused:
|
||||
|
||||
+50
-59
@@ -23,7 +23,6 @@ import dateutil.parser
|
||||
|
||||
from ..exceptions import PatroniFatalException
|
||||
from ..utils import deep_compare, uri
|
||||
from ..tags import Tags
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from ..config import Config
|
||||
@@ -192,11 +191,11 @@ _Version = Union[int, str]
|
||||
_Session = Union[int, float, str, None]
|
||||
|
||||
|
||||
class Member(Tags, NamedTuple('Member',
|
||||
[('version', _Version),
|
||||
('name', str),
|
||||
('session', _Session),
|
||||
('data', Dict[str, Any])])):
|
||||
class Member(NamedTuple('Member',
|
||||
[('version', _Version),
|
||||
('name', str),
|
||||
('session', _Session),
|
||||
('data', Dict[str, Any])])):
|
||||
"""Immutable object (namedtuple) which represents single member of PostgreSQL cluster.
|
||||
|
||||
.. note::
|
||||
@@ -317,10 +316,20 @@ class Member(Tags, NamedTuple('Member',
|
||||
"""The ``tags`` value from :attr:`~Member.data` if defined, otherwise an empty dictionary."""
|
||||
return self.data.get('tags', {})
|
||||
|
||||
@property
|
||||
def nofailover(self) -> bool:
|
||||
"""The value for ``nofailover`` in :attr:`Member`.tags`` if defined, otherwise ``False``."""
|
||||
return self.tags.get('nofailover', False)
|
||||
|
||||
@property
|
||||
def replicatefrom(self) -> Optional[str]:
|
||||
"""The value for ``replicatefrom`` in :attr:`Member`.tags`` if defined."""
|
||||
return self.tags.get('replicatefrom')
|
||||
|
||||
@property
|
||||
def clonefrom(self) -> bool:
|
||||
"""``True`` if both ``clonefrom`` tag is ``True`` and a connection URL is defined."""
|
||||
return super().clonefrom and bool(self.conn_url)
|
||||
return self.tags.get('clonefrom', False) and bool(self.conn_url)
|
||||
|
||||
@property
|
||||
def state(self) -> str:
|
||||
@@ -451,7 +460,7 @@ class Leader(NamedTuple):
|
||||
|
||||
|
||||
class Failover(NamedTuple):
|
||||
"""Immutable object (namedtuple) which represents failover key.
|
||||
"""Immutable object (namedtuple) representing configuration information required for failover/switchover capability.
|
||||
|
||||
:ivar version: version of the object.
|
||||
:ivar leader: name of the leader. If value isn't empty we treat it as a switchover from the specified node.
|
||||
@@ -547,13 +556,6 @@ class Failover(NamedTuple):
|
||||
"""
|
||||
return int(bool(self.leader)) + int(bool(self.candidate))
|
||||
|
||||
@property
|
||||
def is_switchover(self) -> bool:
|
||||
return bool(self.leader)
|
||||
|
||||
@property
|
||||
def is_failover(self) -> bool:
|
||||
return not self.is_switchover
|
||||
|
||||
class ClusterConfig(NamedTuple):
|
||||
"""Immutable object (namedtuple) which represents cluster configuration.
|
||||
@@ -910,16 +912,8 @@ class Cluster(NamedTuple('Cluster',
|
||||
|
||||
@property
|
||||
def __permanent_slots(self) -> Dict[str, Union[Dict[str, Any], Any]]:
|
||||
"""Dictionary of permanent replication slots with their known LSN."""
|
||||
ret = deepcopy(self.config.permanent_slots if self.config else {})
|
||||
# If primary reported flush LSN for permanent slots we want to enrich our structure with it
|
||||
for name, lsn in (self.slots or {}).items():
|
||||
if name in ret:
|
||||
if not ret[name]:
|
||||
ret[name] = {}
|
||||
if isinstance(ret[name], dict):
|
||||
ret[name]['lsn'] = lsn
|
||||
return ret
|
||||
"""Dictionary of permanent replication slots."""
|
||||
return self.config and self.config.permanent_slots or {}
|
||||
|
||||
@property
|
||||
def __permanent_physical_slots(self) -> Dict[str, Any]:
|
||||
@@ -944,6 +938,7 @@ class Cluster(NamedTuple('Cluster',
|
||||
|
||||
Will log an error if:
|
||||
|
||||
* Conflicting slot names between members are found
|
||||
* Any logical slots are disabled, due to version compatibility, and *show_error* is ``True``.
|
||||
|
||||
:param my_name: name of this node.
|
||||
@@ -956,9 +951,21 @@ class Cluster(NamedTuple('Cluster',
|
||||
|
||||
:returns: final dictionary of slot names, after merging with permanent slots and performing sanity checks.
|
||||
"""
|
||||
slots: Dict[str, Dict[str, str]] = self._get_members_slots(my_name, role)
|
||||
permanent_slots: Dict[str, Any] = self._get_permanent_slots(is_standby_cluster, role, nofailover)
|
||||
slot_members: List[str] = self._get_slot_members(my_name, role)
|
||||
|
||||
slots: Dict[str, Dict[str, str]] = {slot_name_from_member_name(name): {'type': 'physical'}
|
||||
for name in slot_members}
|
||||
|
||||
if len(slots) < len(slot_members):
|
||||
# Find which names are conflicting for a nicer error message
|
||||
slot_conflicts: Dict[str, List[str]] = defaultdict(list)
|
||||
for name in slot_members:
|
||||
slot_conflicts[slot_name_from_member_name(name)].append(name)
|
||||
logger.error("Following cluster members share a replication slot name: %s",
|
||||
"; ".join(f"{', '.join(v)} map to {k}"
|
||||
for k, v in slot_conflicts.items() if len(v) > 1))
|
||||
|
||||
permanent_slots: Dict[str, Any] = self._get_permanent_slots(is_standby_cluster, role, nofailover)
|
||||
disabled_permanent_logical_slots: List[str] = self._merge_permanent_slots(
|
||||
slots, permanent_slots, my_name, major_version)
|
||||
|
||||
@@ -1018,7 +1025,7 @@ class Cluster(NamedTuple('Cluster',
|
||||
return disabled_permanent_logical_slots
|
||||
|
||||
def _get_permanent_slots(self, is_standby_cluster: bool, role: str, nofailover: bool) -> Dict[str, Any]:
|
||||
"""Get configured permanent replication slots.
|
||||
"""Get configured permanent slot names.
|
||||
|
||||
.. note::
|
||||
Permanent replication slots are only considered if ``use_slots`` configuration is enabled.
|
||||
@@ -1044,48 +1051,35 @@ class Cluster(NamedTuple('Cluster',
|
||||
|
||||
return self.__permanent_slots if role in ('master', 'primary') else self.__permanent_logical_slots
|
||||
|
||||
def _get_members_slots(self, my_name: str, role: str) -> Dict[str, Dict[str, str]]:
|
||||
"""Get physical replication slots configuration for members that sourcing from this node.
|
||||
def _get_slot_members(self, my_name: str, role: str) -> List[str]:
|
||||
"""Get a list of member names that have replication slots sourcing from this node.
|
||||
|
||||
If the ``replicatefrom`` tag is set on the member - we should not create the replication slot for it on
|
||||
the current primary, because that member would replicate from elsewhere. We still create the slot if
|
||||
the ``replicatefrom`` destination member is currently not a member of the cluster (fallback to the
|
||||
primary), or if ``replicatefrom`` destination member happens to be the current primary.
|
||||
|
||||
Will log an error if:
|
||||
|
||||
* Conflicting slot names between members are found
|
||||
|
||||
:param my_name: name of this node.
|
||||
:param role: role of this node, if this is a ``primary`` or ``standby_leader`` return list of members
|
||||
replicating from this node. If not then return a list of members replicating as cascaded
|
||||
replicas from this node.
|
||||
|
||||
:returns: dictionary of physical replication slots that should exist on a given node.
|
||||
:returns: list of member names.
|
||||
"""
|
||||
if not self.use_slots:
|
||||
return {}
|
||||
|
||||
# we always want to exclude the member with our name from the list
|
||||
members = filter(lambda m: m.name != my_name, self.members)
|
||||
return []
|
||||
|
||||
if role in ('master', 'primary', 'standby_leader'):
|
||||
members = [m for m in members if m.replicatefrom is None
|
||||
or m.replicatefrom == my_name or not self.has_member(m.replicatefrom)]
|
||||
slot_members = [m.name for m in self.members
|
||||
if m.name != my_name
|
||||
and (m.replicatefrom is None
|
||||
or m.replicatefrom == my_name
|
||||
or not self.has_member(m.replicatefrom))]
|
||||
else:
|
||||
# only manage slots for replicas that replicate from this one, except for the leader among them
|
||||
members = [m for m in members if m.replicatefrom == my_name and m.name != self.leader_name]
|
||||
|
||||
slots = {slot_name_from_member_name(m.name): {'type': 'physical'} for m in members}
|
||||
if len(slots) < len(members):
|
||||
# Find which names are conflicting for a nicer error message
|
||||
slot_conflicts: Dict[str, List[str]] = defaultdict(list)
|
||||
for member in members:
|
||||
slot_conflicts[slot_name_from_member_name(member.name)].append(member.name)
|
||||
logger.error("Following cluster members share a replication slot name: %s",
|
||||
"; ".join(f"{', '.join(v)} map to {k}"
|
||||
for k, v in slot_conflicts.items() if len(v) > 1))
|
||||
return slots
|
||||
slot_members = [m.name for m in self.members
|
||||
if m.replicatefrom == my_name and m.name != self.leader_name]
|
||||
return slot_members
|
||||
|
||||
def has_permanent_logical_slots(self, my_name: str, nofailover: bool, major_version: int = 110000) -> bool:
|
||||
"""Check if the given member node has permanent ``logical`` replication slots configured.
|
||||
@@ -1764,29 +1758,26 @@ class AbstractDCS(abc.ABC):
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def _delete_leader(self, leader: Leader) -> bool:
|
||||
def _delete_leader(self) -> bool:
|
||||
"""Remove leader key from DCS.
|
||||
|
||||
This method should remove leader key if current instance is the leader.
|
||||
|
||||
:param leader: :class:`Leader` object with information about the leader.
|
||||
|
||||
:returns: ``True`` if successfully committed to DCS.
|
||||
"""
|
||||
|
||||
def delete_leader(self, leader: Optional[Leader], last_lsn: Optional[int] = None) -> bool:
|
||||
def delete_leader(self, last_lsn: Optional[int] = None) -> bool:
|
||||
"""Update ``optime/leader`` and voluntarily remove leader key from DCS.
|
||||
|
||||
This method should remove leader key if current instance is the leader.
|
||||
|
||||
:param leader: :class:`Leader` object with information about the leader.
|
||||
:param last_lsn: latest checkpoint location in bytes.
|
||||
|
||||
:returns: boolean result of called abstract :meth:`~AbstractDCS._delete_leader`.
|
||||
"""
|
||||
if last_lsn:
|
||||
self.write_status({self._OPTIME: last_lsn})
|
||||
return bool(leader) and self._delete_leader(leader)
|
||||
return self._delete_leader()
|
||||
|
||||
@abc.abstractmethod
|
||||
def cancel_initialization(self) -> bool:
|
||||
|
||||
+6
-32
@@ -141,36 +141,6 @@ class HTTPClient(object):
|
||||
class ConsulClient(base.Consul):
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
"""
|
||||
Consul client with Patroni customisations.
|
||||
|
||||
.. note::
|
||||
|
||||
Parameters, *token*, *cert* and *ca_cert* are not passed to the parent class :class:`consul.base.Consul`.
|
||||
|
||||
Original class documentation,
|
||||
|
||||
*token* is an optional ``ACL token``. If supplied it will be used by
|
||||
default for all requests made with this client session. It's still
|
||||
possible to override this token by passing a token explicitly for a
|
||||
request.
|
||||
|
||||
*consistency* sets the consistency mode to use by default for all reads
|
||||
that support the consistency option. It's still possible to override
|
||||
this by passing explicitly for a given request. *consistency* can be
|
||||
either 'default', 'consistent' or 'stale'.
|
||||
|
||||
*dc* is the datacenter that this agent will communicate with.
|
||||
By default, the datacenter of the host is used.
|
||||
|
||||
*verify* is whether to verify the SSL certificate for HTTPS requests
|
||||
|
||||
*cert* client side certificates for HTTPS requests
|
||||
|
||||
:param args: positional arguments to pass to :class:`consul.base.Consul`
|
||||
:param kwargs: keyword arguments, with *cert*, *ca_cert* and *token* removed, passed to
|
||||
:class:`consul.base.Consul`
|
||||
"""
|
||||
self._cert = kwargs.pop('cert', None)
|
||||
self._ca_cert = kwargs.pop('ca_cert', None)
|
||||
self.token = kwargs.get('token')
|
||||
@@ -673,8 +643,12 @@ class Consul(AbstractDCS):
|
||||
return self._client.kv.put(self.history_path, value)
|
||||
|
||||
@catch_consul_errors
|
||||
def _delete_leader(self, leader: Leader) -> bool:
|
||||
return self._client.kv.delete(self.leader_path, cas=int(leader.version))
|
||||
def _delete_leader(self) -> bool:
|
||||
cluster = self.cluster
|
||||
if cluster and isinstance(cluster.leader, Leader) and\
|
||||
cluster.leader.name == self._name and isinstance(cluster.leader.version, int):
|
||||
return self._client.kv.delete(self.leader_path, cas=cluster.leader.version)
|
||||
return True
|
||||
|
||||
@catch_consul_errors
|
||||
def set_sync_state_value(self, value: str, version: Optional[int] = None) -> Union[int, bool]:
|
||||
|
||||
+1
-1
@@ -809,7 +809,7 @@ class Etcd(AbstractEtcd):
|
||||
return bool(self.retry(self._client.write, self.initialize_path, sysid, prevExist=(not create_new)))
|
||||
|
||||
@catch_etcd_errors
|
||||
def _delete_leader(self, leader: Leader) -> bool:
|
||||
def _delete_leader(self) -> bool:
|
||||
return bool(self._client.delete(self.leader_path, prevValue=self._name))
|
||||
|
||||
@catch_etcd_errors
|
||||
|
||||
@@ -205,7 +205,7 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
|
||||
|
||||
def __init__(self, config: Dict[str, Any], dns_resolver: DnsCachingResolver, cache_ttl: int = 300) -> None:
|
||||
self._token = None
|
||||
self._cluster_version: Tuple[int] = tuple()
|
||||
self._cluster_version: Tuple[int, ...] = tuple()
|
||||
super(Etcd3Client, self).__init__({**config, 'version_prefix': '/v3beta'}, dns_resolver, cache_ttl)
|
||||
|
||||
try:
|
||||
@@ -912,10 +912,11 @@ class Etcd3(AbstractEtcd):
|
||||
return self.retry(self._client.put, self.initialize_path, sysid, create_revision='0' if create_new else None)
|
||||
|
||||
@catch_etcd_errors
|
||||
def _delete_leader(self, leader: Leader) -> bool:
|
||||
fields = build_range_request(self.leader_path)
|
||||
compare = {'key': fields['key'], 'target': 'VALUE', 'value': base64_encode(self._name)}
|
||||
return bool(self._client.txn(compare, {'request_delete_range': fields}))
|
||||
def _delete_leader(self) -> bool:
|
||||
cluster = self.cluster
|
||||
if cluster and isinstance(cluster.leader, Leader) and cluster.leader.name == self._name:
|
||||
return self._client.deleterange(self.leader_path, mod_revision=cluster.leader.version)
|
||||
return True
|
||||
|
||||
@catch_etcd_errors
|
||||
def cancel_initialization(self) -> bool:
|
||||
|
||||
@@ -836,7 +836,7 @@ class Kubernetes(AbstractDCS):
|
||||
self._api.configure_timeouts(self.loop_wait, self._retry.deadline, self.ttl)
|
||||
|
||||
# retriable_http_codes supposed to be either int, list of integers or comma-separated string with integers.
|
||||
retriable_http_codes = config.get('retriable_http_codes', [])
|
||||
retriable_http_codes: Union[str, List[Union[str, int]]] = config.get('retriable_http_codes', [])
|
||||
if not isinstance(retriable_http_codes, list):
|
||||
retriable_http_codes = [c.strip() for c in str(retriable_http_codes).split(',')]
|
||||
|
||||
@@ -1312,11 +1312,11 @@ class Kubernetes(AbstractDCS):
|
||||
if cluster and cluster.config and cluster.config.version else None
|
||||
return self.patch_or_create_config({self._INITIALIZE: sysid}, resource_version)
|
||||
|
||||
def _delete_leader(self, leader: Leader) -> bool:
|
||||
def _delete_leader(self) -> bool:
|
||||
"""Unused"""
|
||||
raise NotImplementedError # pragma: no cover
|
||||
|
||||
def delete_leader(self, leader: Optional[Leader], last_lsn: Optional[int] = None) -> bool:
|
||||
def delete_leader(self, last_lsn: Optional[int] = None) -> bool:
|
||||
ret = False
|
||||
kind = self._kinds.get(self.leader_path)
|
||||
if kind and (kind.metadata.annotations or {}).get(self._LEADER) == self._name:
|
||||
|
||||
+1
-1
@@ -446,7 +446,7 @@ class Raft(AbstractDCS):
|
||||
def initialize(self, create_new: bool = True, sysid: str = '') -> bool:
|
||||
return self._sync_obj.set(self.initialize_path, sysid, prevExist=(not create_new)) is not False
|
||||
|
||||
def _delete_leader(self, leader: Leader) -> bool:
|
||||
def _delete_leader(self) -> bool:
|
||||
return self._sync_obj.delete(self.leader_path, prevValue=self._name, timeout=1)
|
||||
|
||||
def cancel_initialization(self) -> bool:
|
||||
|
||||
@@ -89,7 +89,7 @@ class ZooKeeper(AbstractDCS):
|
||||
def __init__(self, config: Dict[str, Any]) -> None:
|
||||
super(ZooKeeper, self).__init__(config)
|
||||
|
||||
hosts = config.get('hosts', [])
|
||||
hosts: Union[str, List[str]] = config.get('hosts', [])
|
||||
if isinstance(hosts, list):
|
||||
hosts = ','.join(hosts)
|
||||
|
||||
@@ -466,7 +466,7 @@ class ZooKeeper(AbstractDCS):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _delete_leader(self, leader: Leader) -> bool:
|
||||
def _delete_leader(self) -> bool:
|
||||
self._client.restart()
|
||||
return True
|
||||
|
||||
|
||||
+143
-195
@@ -20,28 +20,31 @@ from .postgresql.callback_executor import CallbackAction
|
||||
from .postgresql.misc import postgres_version_to_int
|
||||
from .postgresql.postmaster import PostmasterProcess
|
||||
from .postgresql.rewind import Rewind
|
||||
from .tags import Tags
|
||||
from .utils import polling_loop, tzutc
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class _MemberStatus(Tags, NamedTuple('_MemberStatus',
|
||||
[('member', Member),
|
||||
('reachable', bool),
|
||||
('in_recovery', Optional[bool]),
|
||||
('wal_position', int),
|
||||
('data', Dict[str, Any])])):
|
||||
"""Node status distilled from API response.
|
||||
class _MemberStatus(NamedTuple):
|
||||
"""Node status distilled from API response:
|
||||
|
||||
Consists of the following fields:
|
||||
|
||||
:ivar member: :class:`~patroni.dcs.Member` object of the node.
|
||||
:ivar reachable: ``False`` if the node is not reachable or is not responding with correct JSON.
|
||||
:ivar in_recovery: ``False`` if the node is running as a primary (`if pg_is_in_recovery() == true`).
|
||||
:ivar wal_position: maximum value of ``replayed_location`` or ``received_location`` from JSON.
|
||||
:ivar data: the whole JSON response for future usage.
|
||||
member - dcs.Member object of the node
|
||||
reachable - `!False` if the node is not reachable or is not responding with correct JSON
|
||||
in_recovery - `!True` if pg_is_in_recovery() == true
|
||||
dcs_last_seen - timestamp from JSON of last succesful communication with DCS
|
||||
timeline - timeline value from JSON
|
||||
wal_position - maximum value of `replayed_location` or `received_location` from JSON
|
||||
tags - dictionary with values of different tags (i.e. nofailover)
|
||||
watchdog_failed - indicates that watchdog is required by configuration but not available or failed
|
||||
"""
|
||||
member: Member
|
||||
reachable: bool
|
||||
in_recovery: Optional[bool]
|
||||
dcs_last_seen: int
|
||||
timeline: int
|
||||
wal_position: int
|
||||
tags: Dict[str, Any]
|
||||
watchdog_failed: bool
|
||||
|
||||
@classmethod
|
||||
def from_api_response(cls, member: Member, json: Dict[str, Any]) -> '_MemberStatus':
|
||||
@@ -54,34 +57,21 @@ class _MemberStatus(Tags, NamedTuple('_MemberStatus',
|
||||
wal: Dict[str, Any] = json.get('wal') or json['xlog']
|
||||
# abuse difference in primary/replica response format
|
||||
in_recovery = not (bool(wal.get('location')) or json.get('role') in ('master', 'primary'))
|
||||
timeline = json.get('timeline', 0)
|
||||
dcs_last_seen = json.get('dcs_last_seen', 0)
|
||||
lsn = int(in_recovery and max(wal.get('received_location', 0), wal.get('replayed_location', 0)))
|
||||
return cls(member, True, in_recovery, lsn, json)
|
||||
|
||||
@property
|
||||
def tags(self) -> Dict[str, Any]:
|
||||
"""Dictionary with values of different tags (i.e. nofailover)."""
|
||||
return self.data.get('tags', {})
|
||||
|
||||
@property
|
||||
def timeline(self) -> int:
|
||||
"""Timeline value from JSON."""
|
||||
return self.data.get('timeline', 0)
|
||||
|
||||
@property
|
||||
def watchdog_failed(self) -> bool:
|
||||
"""Indicates that watchdog is required by configuration but not available or failed."""
|
||||
return self.data.get('watchdog_failed', False)
|
||||
return cls(member, True, in_recovery, dcs_last_seen, timeline, lsn,
|
||||
json.get('tags', {}), json.get('watchdog_failed', False))
|
||||
|
||||
@classmethod
|
||||
def unknown(cls, member: Member) -> '_MemberStatus':
|
||||
"""Create a new class instance with empty or null values."""
|
||||
return cls(member, False, None, 0, {})
|
||||
return cls(member, False, None, 0, 0, 0, {}, False)
|
||||
|
||||
def failover_limitation(self) -> Optional[str]:
|
||||
"""Returns reason why this node can't promote or None if everything is ok."""
|
||||
if not self.reachable:
|
||||
return 'not reachable'
|
||||
if self.nofailover:
|
||||
if self.tags.get('nofailover', False):
|
||||
return 'not allowed to promote'
|
||||
if self.watchdog_failed:
|
||||
return 'not watchdog capable'
|
||||
@@ -157,8 +147,8 @@ class Ha(object):
|
||||
self.cluster = Cluster.empty()
|
||||
self.global_config = self.patroni.config.get_global_config(None)
|
||||
self.old_cluster = Cluster.empty()
|
||||
self._leader_expiry = 0
|
||||
self._leader_expiry_lock = RLock()
|
||||
self._is_leader = False
|
||||
self._is_leader_lock = RLock()
|
||||
self._failsafe = Failsafe(patroni.dcs)
|
||||
self._was_paused = False
|
||||
self._leader_timeline = None
|
||||
@@ -203,38 +193,12 @@ class Ha(object):
|
||||
return self.global_config.is_standby_cluster
|
||||
|
||||
def is_leader(self) -> bool:
|
||||
""":returns: `True` if the current node is the leader, based on expiration set when it last held the key."""
|
||||
with self._leader_expiry_lock:
|
||||
return self._leader_expiry > time.time()
|
||||
with self._is_leader_lock:
|
||||
return self._is_leader > time.time()
|
||||
|
||||
def set_is_leader(self, value: bool) -> None:
|
||||
"""Update the current node's view of it's own leadership status.
|
||||
|
||||
Will update the expiry timestamp to match the dcs ttl if setting leadership to true,
|
||||
otherwise will set the expiry to the past to immediately invalidate.
|
||||
|
||||
:param value: is the current node the leader.
|
||||
"""
|
||||
with self._leader_expiry_lock:
|
||||
self._leader_expiry = time.time() + self.dcs.ttl if value else 0
|
||||
|
||||
def sync_mode_is_active(self) -> bool:
|
||||
"""Check whether synchronous replication is requested and already active.
|
||||
|
||||
:returns: ``True`` if the primary already put its name into the ``/sync`` in DCS.
|
||||
"""
|
||||
return self.is_synchronous_mode() and not self.cluster.sync.is_empty
|
||||
|
||||
def _get_failover_action_name(self) -> str:
|
||||
"""Return the currently requested manual failover action name or the default ``failover``.
|
||||
|
||||
:returns: :class:`str` representing the manually requested action (``manual failover`` if no leader
|
||||
is specified in the ``/failover`` in DCS, ``switchover`` otherwise) or ``failover`` if
|
||||
``/failover`` is empty.
|
||||
"""
|
||||
if not self.cluster.failover:
|
||||
return 'failover'
|
||||
return 'switchover' if self.cluster.failover.is_switchover else 'manual failover'
|
||||
with self._is_leader_lock:
|
||||
self._is_leader = time.time() + self.dcs.ttl if value else 0
|
||||
|
||||
def load_cluster_from_dcs(self) -> None:
|
||||
cluster = self.dcs.get_cluster()
|
||||
@@ -508,7 +472,7 @@ class Ha(object):
|
||||
if timeout == 0:
|
||||
# We are requested to prefer failing over to restarting primary. But see first if there
|
||||
# is anyone to fail over to.
|
||||
if self.is_failover_possible():
|
||||
if self.is_failover_possible(self.cluster.members):
|
||||
self.watchdog.disable()
|
||||
logger.info("Primary crashed. Failing over.")
|
||||
self.demote('immediate')
|
||||
@@ -608,7 +572,7 @@ class Ha(object):
|
||||
if refresh:
|
||||
self.load_cluster_from_dcs()
|
||||
|
||||
is_leader = self.state_handler.is_primary()
|
||||
is_leader = self.state_handler.is_leader()
|
||||
|
||||
node_to_follow = self._get_node_to_follow(self.cluster)
|
||||
|
||||
@@ -771,13 +735,13 @@ class Ha(object):
|
||||
if cluster_history:
|
||||
self.dcs.set_history_value('[]')
|
||||
elif not cluster_history or cluster_history[-1][0] != primary_timeline - 1 or len(cluster_history[-1]) != 5:
|
||||
cluster_history = {line[0]: line for line in cluster_history}
|
||||
cluster_history_dict: Dict[int, List[Any]] = {line[0]: list(line) for line in cluster_history}
|
||||
history: List[List[Any]] = list(map(list, self.state_handler.get_history(primary_timeline)))
|
||||
if self.cluster.config:
|
||||
history = history[-self.cluster.config.max_timelines_history:]
|
||||
for line in history:
|
||||
# enrich current history with promotion timestamps stored in DCS
|
||||
cluster_history_line = list(cluster_history.get(line[0], []))
|
||||
cluster_history_line = cluster_history_dict.get(line[0], [])
|
||||
if len(line) == 3 and len(cluster_history_line) >= 4 and cluster_history_line[1] == line[1]:
|
||||
line.append(cluster_history_line[3])
|
||||
if len(cluster_history_line) == 5:
|
||||
@@ -796,7 +760,7 @@ class Ha(object):
|
||||
"""
|
||||
if not self.is_paused():
|
||||
if not self.watchdog.is_running and not self.watchdog.activate():
|
||||
if self.state_handler.is_primary():
|
||||
if self.state_handler.is_leader():
|
||||
self.demote('immediate')
|
||||
return 'Demoting self because watchdog could not be activated'
|
||||
else:
|
||||
@@ -812,7 +776,7 @@ class Ha(object):
|
||||
self._async_response.reset()
|
||||
return 'Promotion cancelled because the pre-promote script failed'
|
||||
|
||||
if self.state_handler.is_primary():
|
||||
if self.state_handler.is_leader():
|
||||
# Inform the state handler about its primary role.
|
||||
# It may be unaware of it if postgres is promoted manually.
|
||||
self.state_handler.set_role('master')
|
||||
@@ -862,8 +826,6 @@ class Ha(object):
|
||||
return _MemberStatus.unknown(member)
|
||||
|
||||
def fetch_nodes_statuses(self, members: List[Member]) -> List[_MemberStatus]:
|
||||
if not members:
|
||||
return []
|
||||
pool = ThreadPool(len(members))
|
||||
results = pool.map(self.fetch_node_status, members) # Run API calls on members in parallel
|
||||
pool.close()
|
||||
@@ -922,27 +884,6 @@ class Ha(object):
|
||||
lag = (self.cluster.last_lsn or 0) - wal_position
|
||||
return lag > self.global_config.maximum_lag_on_failover
|
||||
|
||||
def has_members_eligible_to_promote(self, members: List[Member], reference_lsn: int = 0,
|
||||
fast_path: bool = False) -> bool:
|
||||
ret = False
|
||||
cluster_timeline = self.cluster.timeline
|
||||
|
||||
for st in self.fetch_nodes_statuses(members):
|
||||
not_allowed_reason = st.failover_limitation()
|
||||
if not_allowed_reason:
|
||||
logger.info('Member %s is %s', st.member.name, not_allowed_reason)
|
||||
elif fast_path:
|
||||
return True
|
||||
elif reference_lsn and st.wal_position < reference_lsn or \
|
||||
not reference_lsn and self.is_lagging(st.wal_position):
|
||||
logger.info('Member %s exceeds maximum replication lag', st.member.name)
|
||||
elif self.check_timeline() and (not st.timeline or st.timeline < cluster_timeline):
|
||||
logger.info('Timeline %s of member %s is behind the cluster timeline %s',
|
||||
st.timeline, st.member.name, cluster_timeline)
|
||||
else:
|
||||
ret = True
|
||||
return ret
|
||||
|
||||
def _is_healthiest_node(self, members: Collection[Member], check_replication_lag: bool = True) -> bool:
|
||||
"""This method tries to determine whether I am healthy enough to became a new leader candidate or not."""
|
||||
|
||||
@@ -964,38 +905,52 @@ class Ha(object):
|
||||
# Prepare list of nodes to run check against
|
||||
members = [m for m in members if m.name != self.state_handler.name and not m.nofailover and m.api_url]
|
||||
|
||||
for st in self.fetch_nodes_statuses(members):
|
||||
if st.failover_limitation() is None:
|
||||
if st.in_recovery is False:
|
||||
logger.warning('Primary (%s) is still alive', st.member.name)
|
||||
return False
|
||||
if my_wal_position < st.wal_position:
|
||||
logger.info('Wal position of %s is ahead of my wal position', st.member.name)
|
||||
# In synchronous mode the former leader might be still accessible and even be ahead of us.
|
||||
# We should not disqualify himself from the leader race in such a situation.
|
||||
if not self.sync_mode_is_active() or not self.cluster.sync.leader_matches(st.member.name):
|
||||
if members:
|
||||
for st in self.fetch_nodes_statuses(members):
|
||||
if st.failover_limitation() is None:
|
||||
if st.in_recovery is False:
|
||||
logger.warning('Primary (%s) is still alive', st.member.name)
|
||||
return False
|
||||
logger.info('Ignoring the former leader being ahead of us')
|
||||
if my_wal_position < st.wal_position:
|
||||
logger.info('Wal position of %s is ahead of my wal position', st.member.name)
|
||||
# In synchronous mode the former leader might be still accessible and even be ahead of us.
|
||||
# We should not disqualify himself from the leader race in such a situation.
|
||||
if not self.is_synchronous_mode() or self.cluster.sync.is_empty\
|
||||
or not self.cluster.sync.leader_matches(st.member.name):
|
||||
return False
|
||||
logger.info('Ignoring the former leader being ahead of us')
|
||||
return True
|
||||
|
||||
def is_failover_possible(self, *, cluster_lsn: int = 0, exclude_failover_candidate: bool = False) -> bool:
|
||||
"""Checks whether any of the cluster members is allowed to promote and is healthy enough for that.
|
||||
def is_failover_possible(self, members: List[Member], check_synchronous: Optional[bool] = True,
|
||||
cluster_lsn: Optional[int] = 0) -> bool:
|
||||
"""Checks whether one of the members from the list can possibly win the leader race.
|
||||
|
||||
:param cluster_lsn: to calculate replication lag and exclude member if it is lagging.
|
||||
:param exclude_failover_candidate: if ``True``, exclude :attr:`failover.candidate` from the members
|
||||
list against which the failover possibility checks are run.
|
||||
:returns: `True` if there are members eligible to become the new leader.
|
||||
:param members: list of members to check
|
||||
:param check_synchronous: consider only members that are known to be listed in /sync key when sync replication.
|
||||
:param cluster_lsn: to calculate replication lag and exclude member if it is laggin
|
||||
:returns: `True` if there are members eligible to be the new leader
|
||||
"""
|
||||
candidates = self.get_failover_candidates(exclude_failover_candidate)
|
||||
|
||||
action = self._get_failover_action_name()
|
||||
if self.is_synchronous_mode() and self.cluster.failover and self.cluster.failover.candidate and not candidates:
|
||||
logger.warning('%s candidate=%s does not match with sync_standbys=%s',
|
||||
action.title(), self.cluster.failover.candidate, self.cluster.sync.sync_standby)
|
||||
elif not candidates:
|
||||
logger.warning('%s: candidates list is empty', action)
|
||||
|
||||
return self.has_members_eligible_to_promote(candidates, cluster_lsn)
|
||||
ret = False
|
||||
cluster_timeline = self.cluster.timeline
|
||||
members = [m for m in members if m.name != self.state_handler.name and not m.nofailover and m.api_url]
|
||||
if check_synchronous and self.is_synchronous_mode() and not self.cluster.sync.is_empty:
|
||||
members = [m for m in members if self.cluster.sync.matches(m.name)]
|
||||
if members:
|
||||
for st in self.fetch_nodes_statuses(members):
|
||||
not_allowed_reason = st.failover_limitation()
|
||||
if not_allowed_reason:
|
||||
logger.info('Member %s is %s', st.member.name, not_allowed_reason)
|
||||
elif cluster_lsn and st.wal_position < cluster_lsn or\
|
||||
not cluster_lsn and self.is_lagging(st.wal_position):
|
||||
logger.info('Member %s exceeds maximum replication lag', st.member.name)
|
||||
elif self.check_timeline() and (not st.timeline or st.timeline < cluster_timeline):
|
||||
logger.info('Timeline %s of member %s is behind the cluster timeline %s',
|
||||
st.timeline, st.member.name, cluster_timeline)
|
||||
else:
|
||||
ret = True
|
||||
else:
|
||||
logger.warning('manual failover: members list is empty')
|
||||
return ret
|
||||
|
||||
def manual_failover_process_no_leader(self) -> Optional[bool]:
|
||||
"""Handles manual failover/switchover when the old leader already stepped down.
|
||||
@@ -1006,18 +961,15 @@ class Ha(object):
|
||||
failover = self.cluster.failover
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
assert failover is not None
|
||||
|
||||
action = self._get_failover_action_name()
|
||||
|
||||
if failover.candidate: # manual failover/switchover to specific member
|
||||
if failover.candidate == self.state_handler.name: # manual failover/switchover to me
|
||||
if failover.candidate: # manual failover to specific member
|
||||
if failover.candidate == self.state_handler.name: # manual failover to me
|
||||
return True
|
||||
elif self.is_paused():
|
||||
# Remove failover key if the node to failover has terminated to avoid waiting for it indefinitely
|
||||
# In order to avoid attempts to delete this key from all nodes only the primary is allowed to do it.
|
||||
if not self.cluster.get_member(failover.candidate, fallback_to_leader=False)\
|
||||
and self.state_handler.is_primary():
|
||||
logger.warning("%s: removing failover key because failover candidate is not running", action)
|
||||
and self.state_handler.is_leader():
|
||||
logger.warning("manual failover: removing failover key because failover candidate is not running")
|
||||
self.dcs.manual_failover('', '', version=failover.version)
|
||||
return None
|
||||
return False
|
||||
@@ -1033,21 +985,22 @@ class Ha(object):
|
||||
st = self.fetch_node_status(member)
|
||||
not_allowed_reason = st.failover_limitation()
|
||||
if not_allowed_reason is None: # node is healthy
|
||||
logger.info('%s: to %s, i am %s', action, st.member.name, self.state_handler.name)
|
||||
logger.info('manual failover: to %s, i am %s', st.member.name, self.state_handler.name)
|
||||
return False
|
||||
# we wanted to failover/switchover to specific member but it is not healthy
|
||||
logger.warning('%s: member %s is %s', action, st.member.name, not_allowed_reason)
|
||||
# we wanted to failover to specific member but it is not healthy
|
||||
logger.warning('manual failover: member %s is %s', st.member.name, not_allowed_reason)
|
||||
|
||||
# at this point we should consider all members as a candidates for failover/switchover
|
||||
# at this point we should consider all members as a candidates for failover
|
||||
# i.e. we assume that failover.candidate is None
|
||||
elif self.is_paused():
|
||||
return False
|
||||
|
||||
# try to pick some other members for switchover and check that they are healthy
|
||||
if failover.is_switchover:
|
||||
# try to pick some other members to failover and check that they are healthy
|
||||
if failover.leader:
|
||||
if self.state_handler.name == failover.leader: # I was the leader
|
||||
# exclude desired member which is unhealthy if it was specified
|
||||
if self.is_failover_possible(exclude_failover_candidate=bool(failover.candidate)):
|
||||
# exclude me and desired member which is unhealthy (failover.candidate can be None)
|
||||
members = [m for m in self.cluster.members if m.name not in (failover.candidate, failover.leader)]
|
||||
if self.is_failover_possible(members): # check that there are healthy members
|
||||
return False
|
||||
else: # I was the leader and it looks like currently I am the only healthy member
|
||||
return True
|
||||
@@ -1075,7 +1028,7 @@ class Ha(object):
|
||||
if ret is not None: # continue if we just deleted the stale failover key as a leader
|
||||
return ret
|
||||
|
||||
if self.state_handler.is_primary():
|
||||
if self.state_handler.is_leader():
|
||||
if self.is_paused():
|
||||
# in pause leader is the healthiest only when no initialize or sysid matches with initialize!
|
||||
return not self.cluster.initialize or self.state_handler.sysid == self.cluster.initialize
|
||||
@@ -1101,8 +1054,8 @@ class Ha(object):
|
||||
|
||||
if self.cluster.failover:
|
||||
# When doing a switchover in synchronous mode only synchronous nodes and former leader are allowed to race
|
||||
if self.cluster.failover.is_switchover and self.sync_mode_is_active() \
|
||||
and not self.cluster.sync.matches(self.state_handler.name, True):
|
||||
if self.is_synchronous_mode() and self.cluster.failover.leader and \
|
||||
not self.cluster.sync.is_empty and not self.cluster.sync.matches(self.state_handler.name, True):
|
||||
return False
|
||||
return self.manual_failover_process_no_leader() or False
|
||||
|
||||
@@ -1123,7 +1076,7 @@ class Ha(object):
|
||||
all_known_members += self.cluster.members
|
||||
|
||||
# When in sync mode, only last known primary and sync standby are allowed to promote automatically.
|
||||
if self.sync_mode_is_active():
|
||||
if self.is_synchronous_mode() and not self.cluster.sync.is_empty:
|
||||
if not self.cluster.sync.matches(self.state_handler.name, True):
|
||||
return False
|
||||
# pick between synchronous candidates so we minimize unnecessary failovers/demotions
|
||||
@@ -1136,7 +1089,7 @@ class Ha(object):
|
||||
|
||||
def _delete_leader(self, last_lsn: Optional[int] = None) -> None:
|
||||
self.set_is_leader(False)
|
||||
self.dcs.delete_leader(self.cluster.leader, last_lsn)
|
||||
self.dcs.delete_leader(last_lsn)
|
||||
self.dcs.reset_cluster()
|
||||
|
||||
def release_leader_key_voluntarily(self, last_lsn: Optional[int] = None) -> None:
|
||||
@@ -1176,7 +1129,9 @@ class Ha(object):
|
||||
# It could happen if Postgres is still archiving the backlog of WAL files.
|
||||
# If we know that there are replicas that received the shutdown checkpoint
|
||||
# location, we can remove the leader key and allow them to start leader race.
|
||||
if self.is_failover_possible(cluster_lsn=checkpoint_location):
|
||||
|
||||
# for a manual failover/switchover with a candidate, we should check the requested candidate only
|
||||
if self.is_failover_possible(self.get_failover_candidates(), cluster_lsn=checkpoint_location):
|
||||
self.state_handler.set_role('demoted')
|
||||
with self._async_executor:
|
||||
self.release_leader_key_voluntarily(checkpoint_location)
|
||||
@@ -1266,35 +1221,36 @@ class Ha(object):
|
||||
|
||||
:returns: action message if demote was initiated, None if no action was taken"""
|
||||
failover = self.cluster.failover
|
||||
# if there is no failover key or
|
||||
# I am holding the lock but am not primary = I am the standby leader,
|
||||
# then do nothing
|
||||
if not failover or (self.is_paused() and not self.state_handler.is_primary()):
|
||||
if not failover or (self.is_paused() and not self.state_handler.is_leader()):
|
||||
return
|
||||
|
||||
action = self._get_failover_action_name()
|
||||
bare_action = action.replace('manual ', '')
|
||||
|
||||
# it is not the time for the scheduled switchover yet, do nothing
|
||||
if (failover.scheduled_at and not
|
||||
self.should_run_scheduled_action(bare_action, failover.scheduled_at, lambda:
|
||||
self.should_run_scheduled_action("failover", failover.scheduled_at, lambda:
|
||||
self.dcs.manual_failover('', '', version=failover.version))):
|
||||
return
|
||||
|
||||
if not failover.leader or failover.leader == self.state_handler.name:
|
||||
if not failover.candidate or failover.candidate != self.state_handler.name:
|
||||
if not failover.candidate and self.is_paused():
|
||||
logger.warning('%s is possible only to a specific candidate in a paused state', action.title())
|
||||
elif self.is_failover_possible():
|
||||
ret = self._async_executor.try_run_async(f'{action}: demote', self.demote, ('graceful',))
|
||||
return ret or f'{action}: demoting myself'
|
||||
logger.warning('Failover is possible only to a specific candidate in a paused state')
|
||||
else:
|
||||
logger.warning('%s: no healthy members found, %s is not possible',
|
||||
action, bare_action)
|
||||
if self.is_synchronous_mode():
|
||||
members = self.get_failover_candidates(check_sync=True)
|
||||
if failover.candidate and not members:
|
||||
logger.warning('Failover candidate=%s does not match with sync_standbys=%s',
|
||||
failover.candidate, self.cluster.sync.sync_standby)
|
||||
else:
|
||||
members = self.get_failover_candidates()
|
||||
if self.is_failover_possible(members, False): # check that there are healthy members
|
||||
ret = self._async_executor.try_run_async('manual failover: demote', self.demote, ('graceful',))
|
||||
return ret or 'manual failover: demoting myself'
|
||||
else:
|
||||
logger.warning('manual failover: no healthy members found, failover is not possible')
|
||||
else:
|
||||
logger.warning('%s: I am already the leader, no need to %s', action, bare_action)
|
||||
logger.warning('manual failover: I am already the leader, no need to failover')
|
||||
else:
|
||||
logger.warning('%s: leader name does not match: %s != %s', action, failover.leader, self.state_handler.name)
|
||||
logger.warning('manual failover: leader name does not match: %s != %s',
|
||||
failover.leader, self.state_handler.name)
|
||||
|
||||
logger.info('Cleaning up failover key')
|
||||
self.dcs.manual_failover('', '', version=failover.version)
|
||||
@@ -1343,7 +1299,7 @@ class Ha(object):
|
||||
|
||||
def process_healthy_cluster(self) -> str:
|
||||
if self.has_lock():
|
||||
if self.is_paused() and not self.state_handler.is_primary():
|
||||
if self.is_paused() and not self.state_handler.is_leader():
|
||||
if self.cluster.failover and self.cluster.failover.candidate == self.state_handler.name:
|
||||
return 'waiting to become primary after promote...'
|
||||
|
||||
@@ -1351,7 +1307,6 @@ class Ha(object):
|
||||
self._delete_leader()
|
||||
return 'removed leader lock because postgres is not running as primary'
|
||||
|
||||
# update lock to avoid split-brain
|
||||
if self.update_lock(True):
|
||||
msg = self.process_manual_failover_from_leader()
|
||||
if msg is not None:
|
||||
@@ -1376,7 +1331,7 @@ class Ha(object):
|
||||
else:
|
||||
# Either there is no connection to DCS or someone else acquired the lock
|
||||
logger.error('failed to update leader lock')
|
||||
if self.state_handler.is_primary():
|
||||
if self.state_handler.is_leader():
|
||||
if self.is_paused():
|
||||
return 'continue to run as primary after failing to update leader lock in DCS'
|
||||
self.demote('immediate-nolock')
|
||||
@@ -1549,7 +1504,7 @@ class Ha(object):
|
||||
if self.has_lock() and self.update_lock():
|
||||
if self._async_executor.scheduled_action == 'doing crash recovery in a single user mode':
|
||||
time_left = self.global_config.primary_start_timeout - (time.time() - self._crash_recovery_started)
|
||||
if time_left <= 0 and self.is_failover_possible():
|
||||
if time_left <= 0 and self.is_failover_possible(self.cluster.members):
|
||||
logger.info("Demoting self because crash recovery is taking too long")
|
||||
self.state_handler.cancellable.cancel(True)
|
||||
self.demote('immediate')
|
||||
@@ -1613,7 +1568,7 @@ class Ha(object):
|
||||
self.cancel_initialization()
|
||||
|
||||
if result is None:
|
||||
if not self.state_handler.is_primary():
|
||||
if not self.state_handler.is_leader():
|
||||
return 'waiting for end of recovery after bootstrap'
|
||||
|
||||
self.state_handler.set_role('master')
|
||||
@@ -1660,7 +1615,7 @@ class Ha(object):
|
||||
time_left = timeout - self.state_handler.time_in_state()
|
||||
|
||||
if time_left <= 0:
|
||||
if self.is_failover_possible():
|
||||
if self.is_failover_possible(self.cluster.members):
|
||||
logger.info("Demoting self because primary startup is taking too long")
|
||||
self.demote('immediate')
|
||||
return 'stopped PostgreSQL because of startup timeout'
|
||||
@@ -1797,7 +1752,7 @@ class Ha(object):
|
||||
elif self.cluster.is_unlocked() and not self.is_paused():
|
||||
# "bootstrap", but data directory is not empty
|
||||
if not self.state_handler.cb_called and self.state_handler.is_running() \
|
||||
and not self.state_handler.is_primary():
|
||||
and not self.state_handler.is_leader():
|
||||
self._join_aborted = True
|
||||
logger.error('No initialize key in DCS and PostgreSQL is running as replica, aborting start')
|
||||
logger.error('Please first start Patroni on the node running as primary')
|
||||
@@ -1840,7 +1795,7 @@ class Ha(object):
|
||||
create_slots = self._sync_replication_slots(False)
|
||||
|
||||
if not self.state_handler.cb_called:
|
||||
if not is_promoting and not self.state_handler.is_primary():
|
||||
if not is_promoting and not self.state_handler.is_leader():
|
||||
self._rewind.trigger_check_diverged_lsn()
|
||||
self.state_handler.call_nowait(CallbackAction.ON_START)
|
||||
|
||||
@@ -1865,7 +1820,7 @@ class Ha(object):
|
||||
|
||||
def _handle_dcs_error(self) -> str:
|
||||
if not self.is_paused() and self.state_handler.is_running():
|
||||
if self.state_handler.is_primary():
|
||||
if self.state_handler.is_leader():
|
||||
if self.is_failsafe_mode() and self.check_failsafe_topology():
|
||||
self.set_is_leader(True)
|
||||
self._failsafe.set_is_active(time.time())
|
||||
@@ -1941,8 +1896,9 @@ class Ha(object):
|
||||
# If we know that there are replicas that received the shutdown checkpoint
|
||||
# location, we can remove the leader key and allow them to start leader race.
|
||||
|
||||
if self.is_failover_possible(cluster_lsn=checkpoint_location):
|
||||
self.dcs.delete_leader(self.cluster.leader, checkpoint_location)
|
||||
# for a manual failover/switchover with a candidate, we should check the requested candidate only
|
||||
if self.is_failover_possible(self.get_failover_candidates(), cluster_lsn=checkpoint_location):
|
||||
self.dcs.delete_leader(checkpoint_location)
|
||||
status['deleted'] = True
|
||||
else:
|
||||
self.dcs.write_leader_optime(checkpoint_location)
|
||||
@@ -1959,7 +1915,7 @@ class Ha(object):
|
||||
if not self.state_handler.is_running():
|
||||
if self.is_leader() and not status['deleted']:
|
||||
checkpoint_location = self.state_handler.latest_checkpoint_location()
|
||||
self.dcs.delete_leader(self.cluster.leader, checkpoint_location)
|
||||
self.dcs.delete_leader(checkpoint_location)
|
||||
self.touch_member()
|
||||
else:
|
||||
# XXX: what about when Patroni is started as the wrong user that has access to the watchdog device
|
||||
@@ -2003,31 +1959,23 @@ class Ha(object):
|
||||
name = member.name if member else 'remote_member:{}'.format(uuid.uuid1())
|
||||
return RemoteMember(name, data)
|
||||
|
||||
def get_failover_candidates(self, exclude_failover_candidate: bool) -> List[Member]:
|
||||
"""Return a list of candidates for either manual or automatic failover.
|
||||
def get_failover_candidates(self, check_sync: bool = False) -> List[Member]:
|
||||
"""Return list of candidates for either manual or automatic failover.
|
||||
|
||||
Exclude non-sync members when in synchronous mode, the current node (its checks are always performed earlier)
|
||||
and the candidate if required. If failover candidate exclusion is not requested and a candidate is specified
|
||||
in the /failover key, return the candidate only.
|
||||
The result is further evaluated in the caller :func:`Ha.is_failover_possible` to check if any member is actually
|
||||
healthy enough and is allowed to poromote.
|
||||
Mainly used to later be passed to ``Ha.is_failover_possible()``.
|
||||
|
||||
:param exclude_failover_candidate: if ``True``, exclude :attr:`failover.candidate` from the candidates.
|
||||
:param check_sync: if ``True``, also check against the sync key members
|
||||
|
||||
:returns: a list of :class:`Member` ojects or an empty list if there is no candidate available.
|
||||
:returns: a list of ``Member`` ojects or an empty list if there is no candidate available
|
||||
"""
|
||||
failover = self.cluster.failover
|
||||
exclude = [self.state_handler.name] + ([failover.candidate] if failover and exclude_failover_candidate else [])
|
||||
|
||||
def is_eligible(node: Member) -> bool:
|
||||
# in synchronous mode we allow failover (not switchover!) to async node
|
||||
if self.sync_mode_is_active() and not self.cluster.sync.matches(node.name)\
|
||||
and not (failover and failover.is_failover):
|
||||
return False
|
||||
# Don't spend time on "nofailover" nodes checking.
|
||||
# We also don't need nodes which we can't query with the api in the list.
|
||||
return node.name not in exclude and \
|
||||
not node.nofailover and bool(node.api_url) and \
|
||||
(not failover or not failover.candidate or node.name == failover.candidate)
|
||||
|
||||
return list(filter(is_eligible, self.cluster.members))
|
||||
if check_sync:
|
||||
# TODO: allow manual failover (=no leader specified) to async node
|
||||
# every sync_standby or the candidate specified if is in sync_standbys
|
||||
return [m for m in self.cluster.members
|
||||
if self.cluster.sync.matches(m.name)
|
||||
and (not failover or not failover.candidate or m.name == failover.candidate)]
|
||||
else:
|
||||
# every member or the candidate specified
|
||||
return [m for m in self.cluster.members
|
||||
if not failover or not failover.candidate or m.name == failover.candidate]
|
||||
|
||||
@@ -1,93 +0,0 @@
|
||||
from enum import Enum
|
||||
from typing import Optional, Tuple, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
import datetime
|
||||
|
||||
from .dcs import Cluster
|
||||
from .ha import Patroni
|
||||
from .utils import ParseScheduleErrors
|
||||
|
||||
from .utils import parse_schedule
|
||||
|
||||
|
||||
class ManualFailoverPrecheckStatus(Enum):
|
||||
FAILOVER_NO_CANDIDATE = ('Failover could be performed only to a specific candidate', 400)
|
||||
SWITCHOVER_NO_LEADER = ('Switchover could be performed only from a specific leader', 400)
|
||||
SCHEDULED_FAILOVER = ("Failover can't be scheduled", 400)
|
||||
SCHEDULED_SWITCHOVER_PAUSE = ("Can't schedule switchover in the paused state", 400)
|
||||
SWITCHOVER_PAUSE_NO_CANDIDATE = ('Switchover is possible only to a specific candidate in a paused state', 400)
|
||||
SWITCHOVER_TO_LEADER = ('Switchover target and source are the same', 400)
|
||||
|
||||
CLUSTER_NO_LEADER = ('Cluster {cluster_name} has no leader', 412)
|
||||
LEADER_NOT_MEMBER = ('Member {leader} is not the leader of cluster {cluster_name}', 412)
|
||||
CANDIDATE_NOT_SYNC_STANDBY = ('candidate name does not match with sync_standby', 412)
|
||||
NO_SYNC_CANDIDATE = ('{action} is not possible: can not find sync_standby', 412)
|
||||
ONLY_LEADER = ('{action} is not possible: cluster does not have members except leader', 412)
|
||||
CANDIDATE_NOT_MEMEBER = ('Member {candidate} does not exist in cluster {cluster_name} or is tagged as nofailover',
|
||||
412)
|
||||
NO_GOOD_CANDIDATES = ('{action} is not possible: no good candidates have been found', 412)
|
||||
|
||||
CHECK_PASSED = ('', None)
|
||||
|
||||
|
||||
class ManualFailover(object):
|
||||
|
||||
def __init__(self, action: str, cluster: 'Cluster',
|
||||
leader: Optional[str], candidate: Optional[str], scheduled: Optional[str],
|
||||
paused: bool = False, sync_mode: bool = False, patroni_obj: Optional['Patroni'] = None) -> None:
|
||||
self.action = action
|
||||
self.cluster = cluster
|
||||
self.leader = leader
|
||||
self.candidate = candidate
|
||||
self.scheduled = scheduled
|
||||
self.paused = paused
|
||||
self.sync_mode = sync_mode
|
||||
self.patroni = patroni_obj
|
||||
|
||||
def parse_scheduled(self) -> Tuple[Optional['ParseScheduleErrors'], Optional['datetime.datetime']]:
|
||||
return parse_schedule(self.scheduled)
|
||||
|
||||
def run_precheck(self) -> ManualFailoverPrecheckStatus:
|
||||
if self.action == 'failover' and not self.candidate:
|
||||
return ManualFailoverPrecheckStatus.FAILOVER_NO_CANDIDATE
|
||||
elif self.action == 'switchover' and not self.leader:
|
||||
return ManualFailoverPrecheckStatus.SWITCHOVER_NO_LEADER
|
||||
|
||||
if self.scheduled:
|
||||
if self.action == 'failover':
|
||||
return ManualFailoverPrecheckStatus.SCHEDULED_FAILOVER
|
||||
elif self.paused:
|
||||
return ManualFailoverPrecheckStatus.SCHEDULED_SWITCHOVER_PAUSE
|
||||
|
||||
if self.paused and not self.candidate:
|
||||
return ManualFailoverPrecheckStatus.SWITCHOVER_PAUSE_NO_CANDIDATE
|
||||
|
||||
if self.leader == self.candidate:
|
||||
return ManualFailoverPrecheckStatus.SWITCHOVER_TO_LEADER
|
||||
|
||||
if self.action == 'switchover':
|
||||
if self.cluster.leader is None or not self.cluster.leader.name:
|
||||
return ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER
|
||||
if self.cluster.leader.name != self.leader:
|
||||
return ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER
|
||||
|
||||
if self.candidate:
|
||||
if self.action == 'switchover' and self.sync_mode and not self.cluster.sync.matches(self.candidate):
|
||||
return ManualFailoverPrecheckStatus.CANDIDATE_NOT_SYNC_STANDBY
|
||||
members = [m for m in self.cluster.members if m.name == self.candidate]
|
||||
if not members:
|
||||
return ManualFailoverPrecheckStatus.CANDIDATE_NOT_MEMEBER
|
||||
elif self.sync_mode:
|
||||
members = [m for m in self.cluster.members if self.cluster.sync.matches(m.name)]
|
||||
if not members:
|
||||
return ManualFailoverPrecheckStatus.NO_SYNC_CANDIDATE
|
||||
else:
|
||||
members = [m for m in self.cluster.members if not self.cluster.leader or m.name != self.cluster.leader.name and m.api_url]
|
||||
if not members:
|
||||
return ManualFailoverPrecheckStatus.ONLY_LEADER
|
||||
|
||||
if self.patroni and not self.patroni.ha.has_members_eligible_to_promote(members, fast_path=True):
|
||||
return ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES
|
||||
|
||||
return ManualFailoverPrecheckStatus.CHECK_PASSED
|
||||
@@ -18,7 +18,7 @@ from .bootstrap import Bootstrap
|
||||
from .callback_executor import CallbackAction, CallbackExecutor
|
||||
from .cancellable import CancellableSubprocess
|
||||
from .config import ConfigHandler, mtime
|
||||
from .connection import ConnectionPool, get_connection_cursor
|
||||
from .connection import Connection, get_connection_cursor
|
||||
from .citus import CitusHandler
|
||||
from .misc import parse_history, parse_lsn, postgres_major_version_to_int
|
||||
from .postmaster import PostmasterProcess
|
||||
@@ -57,8 +57,8 @@ class Postgresql(object):
|
||||
TL_LSN = ("CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0 "
|
||||
"ELSE ('x' || pg_catalog.substr(pg_catalog.pg_{0}file_name("
|
||||
"pg_catalog.pg_current_{0}_{1}()), 1, 8))::bit(32)::int END, " # primary timeline
|
||||
"CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0 ELSE "
|
||||
"pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}{2}_{1}(), '0/0')::bigint END, " # wal(_flush)?_lsn
|
||||
"CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0 "
|
||||
"ELSE pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}_{1}(), '0/0')::bigint END, " # write_lsn
|
||||
"pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), '0/0')::bigint, "
|
||||
"pg_catalog.pg_{0}_{1}_diff(COALESCE(pg_catalog.pg_last_{0}_receive_{1}(), '0/0'), '0/0')::bigint, "
|
||||
"pg_catalog.pg_is_in_recovery() AND pg_catalog.pg_is_{0}_replay_paused()")
|
||||
@@ -79,8 +79,7 @@ class Postgresql(object):
|
||||
self.set_state('stopped')
|
||||
|
||||
self._pending_restart = False
|
||||
self.connection_pool = ConnectionPool()
|
||||
self._connection = self.connection_pool.get('heartbeat')
|
||||
self._connection = Connection()
|
||||
self.citus_handler = CitusHandler(self, config.get('citus'))
|
||||
self.config = ConfigHandler(self, config)
|
||||
self.config.check_directories()
|
||||
@@ -121,9 +120,9 @@ class Postgresql(object):
|
||||
|
||||
if self.is_running(): # we are "joining" already running postgres
|
||||
self.set_state('running')
|
||||
self.set_role('master' if self.is_primary() else 'replica')
|
||||
self.set_role('master' if self.is_leader() else 'replica')
|
||||
# postpone writing postgresql.conf for 12+ because recovery parameters are not yet known
|
||||
if self.major_version < 120000 or self.is_primary():
|
||||
if self.major_version < 120000 or self.is_leader():
|
||||
self.config.write_postgresql_conf()
|
||||
hba_saved = self.config.replace_pg_hba()
|
||||
ident_saved = self.config.replace_pg_ident()
|
||||
@@ -160,11 +159,6 @@ class Postgresql(object):
|
||||
def wal_name(self) -> str:
|
||||
return 'wal' if self._major_version >= 100000 else 'xlog'
|
||||
|
||||
@property
|
||||
def wal_flush(self) -> str:
|
||||
"""For PostgreSQL 9.6 onwards we want to use pg_current_wal_flush_lsn()/pg_current_xlog_flush_location()."""
|
||||
return '_flush' if self._major_version >= 90600 else ''
|
||||
|
||||
@property
|
||||
def lsn_name(self) -> str:
|
||||
return 'lsn' if self._major_version >= 100000 else 'location'
|
||||
@@ -219,7 +213,7 @@ class Postgresql(object):
|
||||
else:
|
||||
extra = "0, NULL, NULL, NULL, NULL, NULL, NULL" + extra
|
||||
|
||||
return ("SELECT " + self.TL_LSN + ", {3}").format(self.wal_name, self.lsn_name, self.wal_flush, extra)
|
||||
return ("SELECT " + self.TL_LSN + ", {2}").format(self.wal_name, self.lsn_name, extra)
|
||||
|
||||
@property
|
||||
def available_gucs(self) -> CaseInsensitiveSet:
|
||||
@@ -278,7 +272,7 @@ class Postgresql(object):
|
||||
|
||||
:returns: 'ok' if PostgreSQL is up, 'reject' if starting up, 'no_resopnse' if not up."""
|
||||
|
||||
r = self.connection_pool.conn_kwargs
|
||||
r = self.config.local_connect_kwargs
|
||||
cmd = [self.pgcommand('pg_isready'), '-p', r['port'], '-d', self._database]
|
||||
|
||||
# Host is not set if we are connecting via default unix socket
|
||||
@@ -329,50 +323,40 @@ class Postgresql(object):
|
||||
def connection(self) -> Union['connection3', 'Connection3[Any]']:
|
||||
return self._connection.get()
|
||||
|
||||
def _query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]:
|
||||
"""Execute *sql* query with *params* and optionally return results.
|
||||
def set_connection_kwargs(self, kwargs: Dict[str, Any]) -> None:
|
||||
self._connection.set_conn_kwargs(kwargs.copy())
|
||||
self.citus_handler.set_conn_kwargs(kwargs.copy())
|
||||
|
||||
:param sql: SQL statement to execute.
|
||||
:param params: parameters to pass.
|
||||
|
||||
:returns: a query response as a list of tuples if there is any.
|
||||
:raises:
|
||||
:exc:`~psycopg.Error` if had issues while executing *sql*.
|
||||
|
||||
:exc:`~patroni.exceptions.PostgresConnectionException`: if had issues while connecting to the database.
|
||||
|
||||
:exc:`~patroni.utils.RetryFailedError`: if it was detected that connection/query failed due to PostgreSQL
|
||||
restart.
|
||||
"""
|
||||
def _query(self, sql: str, *params: Any) -> Union['Cursor[Any]', 'cursor']:
|
||||
"""We are always using the same cursor, therefore this method is not thread-safe!!!
|
||||
You can call it from different threads only if you are holding explicit `AsyncExecutor` lock,
|
||||
because the main thread is always holding this lock when running HA cycle."""
|
||||
cursor = None
|
||||
try:
|
||||
return self._connection.query(sql, *params)
|
||||
except PostgresConnectionException as exc:
|
||||
cursor = self._connection.cursor()
|
||||
cursor.execute(sql.encode('utf-8'), params or None)
|
||||
return cursor
|
||||
except psycopg.Error as e:
|
||||
if cursor and cursor.connection.closed == 0:
|
||||
# When connected via unix socket, psycopg2 can't recoginze 'connection lost'
|
||||
# and leaves `_cursor_holder.connection.closed == 0`, but psycopg2.OperationalError
|
||||
# is still raised (what is correct). It doesn't make sense to continiue with existing
|
||||
# connection and we will close it, to avoid its reuse by the `cursor` method.
|
||||
if isinstance(e, psycopg.OperationalError):
|
||||
self._connection.close()
|
||||
else:
|
||||
raise e
|
||||
if self.state == 'restarting':
|
||||
raise RetryFailedError('cluster is being restarted') from exc
|
||||
raise
|
||||
raise RetryFailedError('cluster is being restarted')
|
||||
raise PostgresConnectionException('connection problems')
|
||||
|
||||
def query(self, sql: str, *params: Any, retry: bool = True) -> List[Tuple[Any, ...]]:
|
||||
"""Execute *sql* query with *params* and optionally return results.
|
||||
|
||||
:param sql: SQL statement to execute.
|
||||
:param params: parameters to pass.
|
||||
:param retry: whether the query should be retried upon failure or given up immediately.
|
||||
|
||||
:returns: a query response as a list of tuples if there is any.
|
||||
:raises:
|
||||
:exc:`~psycopg.Error` if had issues while executing *sql*.
|
||||
|
||||
:exc:`~patroni.exceptions.PostgresConnectionException`: if had issues while connecting to the database.
|
||||
|
||||
:exc:`~patroni.utils.RetryFailedError`: if it was detected that connection/query failed due to PostgreSQL
|
||||
restart or if retry deadline was exceeded.
|
||||
"""
|
||||
if not retry:
|
||||
return self._query(sql, *params)
|
||||
def query(self, sql: str, *args: Any, **kwargs: Any) -> Union['Cursor[Any]', 'cursor']:
|
||||
if not kwargs.get('retry', True):
|
||||
return self._query(sql, *args)
|
||||
try:
|
||||
return self.retry(self._query, sql, *params)
|
||||
except RetryFailedError as exc:
|
||||
raise PostgresConnectionException(str(exc)) from exc
|
||||
return self.retry(self._query, sql, *args)
|
||||
except RetryFailedError as e:
|
||||
raise PostgresConnectionException(str(e))
|
||||
|
||||
def pg_control_exists(self) -> bool:
|
||||
return os.path.isfile(self._pg_control)
|
||||
@@ -450,7 +434,7 @@ class Postgresql(object):
|
||||
def _cluster_info_state_get(self, name: str) -> Optional[Any]:
|
||||
if not self._cluster_info_state:
|
||||
try:
|
||||
result = self._is_leader_retry(self._query, self.cluster_info_query)[0]
|
||||
result = self._is_leader_retry(self._query, self.cluster_info_query).fetchone()
|
||||
cluster_info_state = dict(zip(['timeline', 'wal_position', 'replayed_location',
|
||||
'received_location', 'replay_paused', 'pg_control_timeline',
|
||||
'received_tli', 'slot_name', 'conninfo', 'receiver_state',
|
||||
@@ -500,14 +484,14 @@ class Postgresql(object):
|
||||
""":returns: a result set of 'SELECT * FROM pg_stat_replication'."""
|
||||
return self._cluster_info_state_get('pg_stat_replication') or []
|
||||
|
||||
def replication_state_from_parameters(self, is_primary: bool, receiver_state: Optional[str],
|
||||
def replication_state_from_parameters(self, is_leader: bool, receiver_state: Optional[str],
|
||||
restore_command: Optional[str]) -> Optional[str]:
|
||||
"""Figure out the replication state from input parameters.
|
||||
|
||||
.. note::
|
||||
This method could be only called when Postgres is up, running and queries are successfuly executed.
|
||||
|
||||
:is_primary: `True` is postgres is not running in recovery
|
||||
:is_leader: `True` is postgres is not running in recovery
|
||||
:receiver_state: value from `pg_stat_get_wal_receiver.state` or None if Postgres is older than 9.6
|
||||
:restore_command: value of ``restore_command`` GUC for PostgreSQL 12+ or
|
||||
`postgresql.recovery_conf.restore_command` if it is set in Patroni configuration
|
||||
@@ -516,7 +500,7 @@ class Postgresql(object):
|
||||
- 'streaming' if replica is streaming according to the `pg_stat_wal_receiver` view;
|
||||
- 'in archive recovery' if replica isn't streaming and there is a `restore_command`
|
||||
"""
|
||||
if self._major_version >= 90600 and not is_primary:
|
||||
if self._major_version >= 90600 and not is_leader:
|
||||
if receiver_state == 'streaming':
|
||||
return 'streaming'
|
||||
# For Postgres older than 12 we get `restore_command` from Patroni config, otherwise we check GUC
|
||||
@@ -531,11 +515,11 @@ class Postgresql(object):
|
||||
|
||||
:returns: ``streaming``, ``in archive recovery``, or ``None``
|
||||
"""
|
||||
return self.replication_state_from_parameters(self.is_primary(),
|
||||
return self.replication_state_from_parameters(self.is_leader(),
|
||||
self._cluster_info_state_get('receiver_state'),
|
||||
self._cluster_info_state_get('restore_command'))
|
||||
|
||||
def is_primary(self) -> bool:
|
||||
def is_leader(self) -> bool:
|
||||
try:
|
||||
return bool(self._cluster_info_state_get('timeline'))
|
||||
except PostgresConnectionException:
|
||||
@@ -568,7 +552,7 @@ class Postgresql(object):
|
||||
r'lsn: ([0-9A-Fa-f]+/[0-9A-Fa-f]+), prev ([0-9A-Fa-f]+/[0-9A-Fa-f]+), '
|
||||
r'.*?desc: (.+)', out.decode('utf-8'))
|
||||
if match:
|
||||
return match.groups()
|
||||
return match.group(1), match.group(2), match.group(3), match.group(4)
|
||||
return None, None, None, None
|
||||
|
||||
def latest_checkpoint_location(self) -> Optional[int]:
|
||||
@@ -691,7 +675,7 @@ class Postgresql(object):
|
||||
# the former node, otherwise, we might get a stalled one
|
||||
# after kill -9, which would report incorrect data to
|
||||
# patroni.
|
||||
self.connection_pool.close()
|
||||
self._connection.close()
|
||||
|
||||
if self.is_running():
|
||||
logger.error('Cannot start PostgreSQL because one is already running.')
|
||||
@@ -762,7 +746,7 @@ class Postgresql(object):
|
||||
def checkpoint(self, connect_kwargs: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[float] = None) -> Optional[str]:
|
||||
check_not_is_in_recovery = connect_kwargs is not None
|
||||
connect_kwargs = connect_kwargs or self.connection_pool.conn_kwargs
|
||||
connect_kwargs = connect_kwargs or self.config.local_connect_kwargs
|
||||
for p in ['connect_timeout', 'options']:
|
||||
connect_kwargs.pop(p, None)
|
||||
if timeout:
|
||||
@@ -892,10 +876,11 @@ class Postgresql(object):
|
||||
|
||||
def _wait_for_connection_close(self, postmaster: PostmasterProcess) -> None:
|
||||
try:
|
||||
while postmaster.is_running(): # Need a timeout here?
|
||||
self._connection.query("SELECT 1")
|
||||
time.sleep(STOP_POLLING_INTERVAL)
|
||||
except (psycopg.Error, PostgresConnectionException):
|
||||
with self.connection().cursor() as cur:
|
||||
while postmaster.is_running(): # Need a timeout here?
|
||||
cur.execute("SELECT 1")
|
||||
time.sleep(STOP_POLLING_INTERVAL)
|
||||
except psycopg.Error:
|
||||
pass
|
||||
|
||||
def reload(self, block_callbacks: bool = False) -> bool:
|
||||
@@ -1023,7 +1008,7 @@ class Postgresql(object):
|
||||
return None, None
|
||||
|
||||
@contextmanager
|
||||
def get_replication_connection_cursor(self, host: Optional[str] = None, port: int = 5432,
|
||||
def get_replication_connection_cursor(self, host: Optional[str] = None, port: Union[int, str] = 5432,
|
||||
**kwargs: Any) -> Iterator[Union['cursor', 'Cursor[Any]']]:
|
||||
conn_kwargs = self.config.replication.copy()
|
||||
conn_kwargs.update(host=host, port=int(port) if port else None, user=conn_kwargs.pop('username'),
|
||||
@@ -1182,9 +1167,9 @@ class Postgresql(object):
|
||||
return ret
|
||||
|
||||
@staticmethod
|
||||
def _wal_position(is_primary: bool, wal_position: int,
|
||||
def _wal_position(is_leader: bool, wal_position: int,
|
||||
received_location: Optional[int], replayed_location: Optional[int]) -> int:
|
||||
return wal_position if is_primary else max(received_location or 0, replayed_location or 0)
|
||||
return wal_position if is_leader else max(received_location or 0, replayed_location or 0)
|
||||
|
||||
def timeline_wal_position(self) -> Tuple[int, int, Optional[int]]:
|
||||
# This method could be called from different threads (simultaneously with some other `_query` calls).
|
||||
@@ -1196,21 +1181,31 @@ class Postgresql(object):
|
||||
received_location = self.received_location()
|
||||
pg_control_timeline = self._cluster_info_state_get('pg_control_timeline')
|
||||
else:
|
||||
timeline, wal_position, replayed_location, received_location, _, pg_control_timeline = \
|
||||
self._query(self.cluster_info_query)[0][:6]
|
||||
with self.connection().cursor() as cursor:
|
||||
cursor.execute(self.cluster_info_query.encode('utf-8'))
|
||||
row = cursor.fetchone()
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
assert row is not None
|
||||
(timeline, wal_position, replayed_location, received_location, _, pg_control_timeline) = row[:6]
|
||||
|
||||
wal_position = self._wal_position(bool(timeline), wal_position, received_location, replayed_location)
|
||||
return timeline, wal_position, pg_control_timeline
|
||||
return (timeline, wal_position, pg_control_timeline)
|
||||
|
||||
def postmaster_start_time(self) -> Optional[str]:
|
||||
try:
|
||||
sql = "SELECT " + self.POSTMASTER_START_TIME
|
||||
return self.query(sql, retry=current_thread().ident == self.__thread_ident)[0][0].isoformat(sep=' ')
|
||||
query = "SELECT " + self.POSTMASTER_START_TIME
|
||||
if current_thread().ident == self.__thread_ident:
|
||||
row = self.query(query).fetchone()
|
||||
else:
|
||||
with self.connection().cursor() as cursor:
|
||||
cursor.execute(query)
|
||||
row = cursor.fetchone()
|
||||
return row[0].isoformat(sep=' ') if row else None
|
||||
except psycopg.Error:
|
||||
return None
|
||||
|
||||
def last_operation(self) -> int:
|
||||
return self._wal_position(self.is_primary(), self._cluster_info_state_get('wal_position') or 0,
|
||||
return self._wal_position(self.is_leader(), self._cluster_info_state_get('wal_position') or 0,
|
||||
self.received_location(), self.replayed_location())
|
||||
|
||||
def configure_server_parameters(self) -> None:
|
||||
|
||||
@@ -176,7 +176,7 @@ class Bootstrap(object):
|
||||
"""
|
||||
cmd = config.get('post_bootstrap') or config.get('post_init')
|
||||
if cmd:
|
||||
r = self._postgresql.connection_pool.conn_kwargs
|
||||
r = self._postgresql.config.local_connect_kwargs
|
||||
connstring = self._postgresql.config.format_dsn(r, True)
|
||||
if 'host' not in r:
|
||||
# https://www.postgresql.org/docs/current/static/libpq-pgpass.html
|
||||
|
||||
+25
-15
@@ -6,10 +6,13 @@ from threading import Condition, Event, Thread
|
||||
from urllib.parse import urlparse
|
||||
from typing import Any, Dict, List, Optional, Union, Tuple, TYPE_CHECKING
|
||||
|
||||
from .connection import Connection
|
||||
from ..dcs import CITUS_COORDINATOR_GROUP_ID, Cluster
|
||||
from ..psycopg import connect, quote_ident
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from psycopg import Cursor
|
||||
from psycopg2 import cursor
|
||||
from . import Postgresql
|
||||
|
||||
CITUS_SLOT_NAME_RE = re.compile(r'^citus_shard_(move|split)_slot(_[1-9][0-9]*){2,3}$')
|
||||
@@ -70,10 +73,7 @@ class CitusHandler(Thread):
|
||||
self.daemon = True
|
||||
self._postgresql = postgresql
|
||||
self._config = config
|
||||
if config:
|
||||
self._connection = postgresql.connection_pool.get(
|
||||
'citus', {'dbname': config['database'],
|
||||
'options': '-c statement_timeout=0 -c idle_in_transaction_session_timeout=0'})
|
||||
self._connection = Connection()
|
||||
self._pg_dist_node: Dict[int, PgDistNode] = {} # Cache of pg_dist_node: {groupid: PgDistNode()}
|
||||
self._tasks: List[PgDistNode] = [] # Requests to change pg_dist_node, every task is a `PgDistNode`
|
||||
self._in_flight: Optional[PgDistNode] = None # Reference to the `PgDistNode` being changed in a transaction
|
||||
@@ -93,6 +93,12 @@ class CitusHandler(Thread):
|
||||
def is_worker(self) -> bool:
|
||||
return self.is_enabled() and not self.is_coordinator()
|
||||
|
||||
def set_conn_kwargs(self, kwargs: Dict[str, Any]) -> None:
|
||||
if isinstance(self._config, dict): # self.is_enabled():
|
||||
kwargs.update({'dbname': self._config['database'],
|
||||
'options': '-c statement_timeout=0 -c idle_in_transaction_session_timeout=0'})
|
||||
self._connection.set_conn_kwargs(kwargs)
|
||||
|
||||
def schedule_cache_rebuild(self) -> None:
|
||||
with self._condition:
|
||||
self._schedule_load_pg_dist_node = True
|
||||
@@ -103,10 +109,12 @@ class CitusHandler(Thread):
|
||||
self._tasks[:] = []
|
||||
self._in_flight = None
|
||||
|
||||
def query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]:
|
||||
def query(self, sql: str, *params: Any) -> Union['Cursor[Any]', 'cursor']:
|
||||
try:
|
||||
logger.debug('query(%s, %s)', sql, params)
|
||||
return self._connection.query(sql, *params)
|
||||
cursor = self._connection.cursor()
|
||||
cursor.execute(sql.encode('utf-8'), params or None)
|
||||
return cursor
|
||||
except Exception as e:
|
||||
logger.error('Exception when executing query "%s", (%s): %r', sql, params, e)
|
||||
self._connection.close()
|
||||
@@ -124,13 +132,13 @@ class CitusHandler(Thread):
|
||||
self._schedule_load_pg_dist_node = False
|
||||
|
||||
try:
|
||||
rows = self.query("SELECT nodeid, groupid, nodename, nodeport, noderole"
|
||||
" FROM pg_catalog.pg_dist_node WHERE noderole = 'primary'")
|
||||
cursor = self.query("SELECT nodeid, groupid, nodename, nodeport, noderole"
|
||||
" FROM pg_catalog.pg_dist_node WHERE noderole = 'primary'")
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
with self._condition:
|
||||
self._pg_dist_node = {r[1]: PgDistNode(r[1], r[2], r[3], 'after_promote', r[0]) for r in rows}
|
||||
self._pg_dist_node = {r[1]: PgDistNode(r[1], r[2], r[3], 'after_promote', r[0]) for r in cursor}
|
||||
return True
|
||||
|
||||
def sync_pg_dist_node(self, cluster: Cluster) -> None:
|
||||
@@ -203,8 +211,10 @@ class CitusHandler(Thread):
|
||||
self.query('SELECT pg_catalog.citus_update_node(%s, %s, %s, true, %s)',
|
||||
task.nodeid, task.host, task.port, task.cooldown)
|
||||
elif task.event != 'before_demote':
|
||||
task.nodeid = self.query("SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default')",
|
||||
task.host, task.port, task.group)[0][0]
|
||||
row = self.query("SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default')",
|
||||
task.host, task.port, task.group).fetchone()
|
||||
if row is not None:
|
||||
task.nodeid = row[0]
|
||||
|
||||
def process_task(self, task: PgDistNode) -> bool:
|
||||
"""Updates a single row in `pg_dist_node` table, optionally in a transaction.
|
||||
@@ -355,8 +365,8 @@ class CitusHandler(Thread):
|
||||
if not isinstance(self._config, dict): # self.is_enabled()
|
||||
return
|
||||
|
||||
conn_kwargs = {**self._postgresql.connection_pool.conn_kwargs,
|
||||
'options': '-c synchronous_commit=local -c statement_timeout=0'}
|
||||
conn_kwargs = self._postgresql.config.local_connect_kwargs
|
||||
conn_kwargs['options'] = '-c synchronous_commit=local -c statement_timeout=0'
|
||||
if self._config['database'] != self._postgresql.database:
|
||||
conn = connect(**conn_kwargs)
|
||||
try:
|
||||
@@ -397,14 +407,14 @@ class CitusHandler(Thread):
|
||||
parameters['shared_preload_libraries'] = ','.join(['citus'] + shared_preload_libraries)
|
||||
|
||||
# if not explicitly set Citus overrides max_prepared_transactions to max_connections*2
|
||||
if parameters.get('max_prepared_transactions') == 0:
|
||||
if parameters['max_prepared_transactions'] == 0:
|
||||
parameters['max_prepared_transactions'] = parameters['max_connections'] * 2
|
||||
|
||||
# Resharding in Citus implemented using logical replication
|
||||
parameters['wal_level'] = 'logical'
|
||||
|
||||
def ignore_replication_slot(self, slot: Dict[str, str]) -> bool:
|
||||
if isinstance(self._config, dict) and self._postgresql.is_primary() and\
|
||||
if isinstance(self._config, dict) and self._postgresql.is_leader() and\
|
||||
slot['type'] == 'logical' and slot['database'] == self._config['database']:
|
||||
m = CITUS_SLOT_NAME_RE.match(slot['name'])
|
||||
return bool(m and {'move': 'pgoutput', 'split': 'citus'}.get(m.group(1)) == slot['plugin'])
|
||||
|
||||
@@ -968,32 +968,24 @@ class ConfigHandler(object):
|
||||
return 'localhost' # connection via localhost is preferred
|
||||
return listen_addresses[0].strip() # can't use localhost, take first address from listen_addresses
|
||||
|
||||
@property
|
||||
def local_connect_kwargs(self) -> Dict[str, Any]:
|
||||
ret = self._local_address.copy()
|
||||
# add all of the other connection settings that are available
|
||||
ret.update(self._superuser)
|
||||
# if the "username" parameter is present, it actually needs to be "user"
|
||||
# for connecting to PostgreSQL
|
||||
if 'username' in self._superuser:
|
||||
ret['user'] = self._superuser['username']
|
||||
del ret['username']
|
||||
# ensure certain Patroni configurations are available
|
||||
ret.update({'dbname': self._postgresql.database,
|
||||
'fallback_application_name': 'Patroni',
|
||||
'connect_timeout': 3,
|
||||
'options': '-c statement_timeout=2000'})
|
||||
return ret
|
||||
|
||||
def resolve_connection_addresses(self) -> None:
|
||||
"""Calculates and sets local and remote connection urls and options.
|
||||
|
||||
This method sets:
|
||||
* :attr:`Postgresql.connection_string <patroni.postgresql.Postgresql.connection_string>` attribute, which
|
||||
is later written to the member key in DCS as ``conn_url``.
|
||||
* :attr:`ConfigHandler.local_replication_address` attribute, which is used for replication connections to
|
||||
local postgres.
|
||||
* :attr:`ConnectionPool.conn_kwargs <patroni.postgresql.connection.ConnectionPool.conn_kwargs>` attribute,
|
||||
which is used for superuser connections to local postgres.
|
||||
|
||||
.. note::
|
||||
If there is a valid directory in ``postgresql.parameters.unix_socket_directories`` in the Patroni
|
||||
configuration and ``postgresql.use_unix_socket`` and/or ``postgresql.use_unix_socket_repl``
|
||||
are set to ``True``, we respectively use unix sockets for superuser and replication connections
|
||||
to local postgres.
|
||||
|
||||
If there is a requirement to use unix sockets, but nothing is set in the
|
||||
``postgresql.parameters.unix_socket_directories``, we omit a ``host`` in connection parameters relying
|
||||
on the ability of ``libpq`` to connect via some default unix socket directory.
|
||||
|
||||
If unix sockets are not requested we "switch" to TCP, prefering to use ``localhost`` if it is possible
|
||||
to deduce that Postgres is listening on a local interface address.
|
||||
|
||||
Otherwise we just used the first address specified in the ``listen_addresses`` GUC.
|
||||
"""
|
||||
port = self._server_parameters['port']
|
||||
tcp_local_address = self._get_tcp_local_address()
|
||||
netloc = self._config.get('connect_address') or tcp_local_address + ':' + port
|
||||
@@ -1006,37 +998,21 @@ class ConfigHandler(object):
|
||||
|
||||
tcp_local_address = {'host': tcp_local_address, 'port': port}
|
||||
|
||||
self._local_address = unix_local_address if self._config.get('use_unix_socket') else tcp_local_address
|
||||
self.local_replication_address = unix_local_address\
|
||||
if self._config.get('use_unix_socket_repl') else tcp_local_address
|
||||
|
||||
self._postgresql.connection_string = uri('postgres', netloc, self._postgresql.database)
|
||||
self._postgresql.set_connection_kwargs(self.local_connect_kwargs)
|
||||
|
||||
local_address = unix_local_address if self._config.get('use_unix_socket') else tcp_local_address
|
||||
local_conn_kwargs = {
|
||||
**local_address,
|
||||
**self._superuser,
|
||||
'dbname': self._postgresql.database,
|
||||
'fallback_application_name': 'Patroni',
|
||||
'connect_timeout': 3,
|
||||
'options': '-c statement_timeout=2000'
|
||||
}
|
||||
# if the "username" parameter is present, it actually needs to be "user" for connecting to PostgreSQL
|
||||
if 'username' in local_conn_kwargs:
|
||||
local_conn_kwargs['user'] = local_conn_kwargs.pop('username')
|
||||
# "notify" connection_pool about the "new" local connection address
|
||||
self._postgresql.connection_pool.conn_kwargs = local_conn_kwargs
|
||||
|
||||
def _get_pg_settings(
|
||||
self, names: Collection[str]
|
||||
) -> Dict[str, Tuple[str, str, Optional[str], str, str, Optional[str]]]:
|
||||
def _get_pg_settings(self, names: Collection[str]) -> Dict[Any, Tuple[Any, ...]]:
|
||||
return {r[0]: r for r in self._postgresql.query(('SELECT name, setting, unit, vartype, context, sourcefile'
|
||||
+ ' FROM pg_catalog.pg_settings '
|
||||
+ ' WHERE pg_catalog.lower(name) = ANY(%s)'),
|
||||
[n.lower() for n in names])}
|
||||
|
||||
@staticmethod
|
||||
def _handle_wal_buffers(old_values: Dict[str, Tuple[str, str, Optional[str], str, str, Optional[str]]],
|
||||
changes: CaseInsensitiveDict) -> None:
|
||||
def _handle_wal_buffers(old_values: Dict[Any, Tuple[Any, ...]], changes: CaseInsensitiveDict) -> None:
|
||||
wal_block_size = parse_int(old_values['wal_block_size'][1]) or 8192
|
||||
wal_segment_size = old_values['wal_segment_size']
|
||||
wal_segment_unit = parse_int(wal_segment_size[2], 'B') or 8192 \
|
||||
@@ -1140,10 +1116,10 @@ class ConfigHandler(object):
|
||||
if self._postgresql.major_version >= 90500:
|
||||
time.sleep(1)
|
||||
try:
|
||||
pending_restart = self._postgresql.query(
|
||||
pending_restart = (self._postgresql.query(
|
||||
'SELECT COUNT(*) FROM pg_catalog.pg_settings'
|
||||
' WHERE pg_catalog.lower(name) != ALL(%s) AND pending_restart',
|
||||
[n.lower() for n in self._RECOVERY_PARAMETERS])[0][0] > 0
|
||||
[n.lower() for n in self._RECOVERY_PARAMETERS]).fetchone() or (0,))[0] > 0
|
||||
self._postgresql.set_pending_restart(pending_restart)
|
||||
except Exception as e:
|
||||
logger.warning('Exception %r when running query', e)
|
||||
|
||||
@@ -2,153 +2,45 @@ import logging
|
||||
|
||||
from contextlib import contextmanager
|
||||
from threading import Lock
|
||||
from typing import Any, Dict, Iterator, List, Optional, Union, Tuple, TYPE_CHECKING
|
||||
from typing import Any, Dict, Iterator, Union, TYPE_CHECKING
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from psycopg import Connection, Cursor
|
||||
from psycopg import Connection as Connection3, Cursor
|
||||
from psycopg2 import connection, cursor
|
||||
|
||||
from .. import psycopg
|
||||
from ..exceptions import PostgresConnectionException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class NamedConnection:
|
||||
"""Helper class to manage ``psycopg`` connections from Patroni to PostgreSQL.
|
||||
|
||||
:ivar server_version: PostgreSQL version in integer format where we are connected to.
|
||||
"""
|
||||
|
||||
class Connection(object):
|
||||
server_version: int
|
||||
|
||||
def __init__(self, pool: 'ConnectionPool', name: str, kwargs_override: Optional[Dict[str, Any]]) -> None:
|
||||
"""Create an instance of :class:`NamedConnection` class.
|
||||
|
||||
:param pool: reference to a :class:`ConnectionPool` object.
|
||||
:param name: name of the connection.
|
||||
:param kwargs_override: :class:`dict` object with connection parameters that should be
|
||||
different from default values provided by connection *pool*.
|
||||
"""
|
||||
self._pool = pool
|
||||
self._name = name
|
||||
self._kwargs_override = kwargs_override or {}
|
||||
self._lock = Lock() # used to make sure that only one connection to postgres is established
|
||||
def __init__(self) -> None:
|
||||
self._lock = Lock()
|
||||
self._connection = None
|
||||
self._cursor_holder = None
|
||||
|
||||
@property
|
||||
def _conn_kwargs(self) -> Dict[str, Any]:
|
||||
"""Connection parameters for this :class:`NamedConnection`."""
|
||||
return {**self._pool.conn_kwargs, **self._kwargs_override, 'application_name': f'Patroni {self._name}'}
|
||||
def set_conn_kwargs(self, conn_kwargs: Dict[str, Any]) -> None:
|
||||
self._conn_kwargs = conn_kwargs
|
||||
|
||||
def get(self) -> Union['connection', 'Connection[Any]']:
|
||||
"""Get ``psycopg``/``psycopg2`` connection object.
|
||||
|
||||
.. note::
|
||||
Opens a new connection if necessary.
|
||||
|
||||
:returns: ``psycopg`` or ``psycopg2`` connection object.
|
||||
"""
|
||||
def get(self) -> Union['connection', 'Connection3[Any]']:
|
||||
with self._lock:
|
||||
if not self._connection or self._connection.closed != 0:
|
||||
logger.info("establishing a new patroni %s connection to postgres", self._name)
|
||||
self._connection = psycopg.connect(**self._conn_kwargs)
|
||||
self.server_version = getattr(self._connection, 'server_version', 0)
|
||||
return self._connection
|
||||
|
||||
def query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]:
|
||||
"""Execute a query with parameters and optionally returns a response.
|
||||
|
||||
:param sql: SQL statement to execute.
|
||||
:param params: parameters to pass.
|
||||
|
||||
:returns: a query response as a list of tuples if there is any.
|
||||
:raises:
|
||||
:exc:`~psycopg.Error` if had issues while executing *sql*.
|
||||
|
||||
:exc:`~patroni.exceptions.PostgresConnectionException`: if had issues while connecting to the database.
|
||||
"""
|
||||
cursor = None
|
||||
try:
|
||||
with self.get().cursor() as cursor:
|
||||
cursor.execute(sql.encode('utf-8'), params or None)
|
||||
return cursor.fetchall() if cursor.rowcount and cursor.rowcount > 0 else []
|
||||
except psycopg.Error as exc:
|
||||
if cursor and cursor.connection.closed == 0:
|
||||
# When connected via unix socket, psycopg2 can't recoginze 'connection lost' and leaves
|
||||
# `self._connection.closed == 0`, but the generic exception is raised. It doesn't make
|
||||
# sense to continue with existing connection and we will close it, to avoid its reuse.
|
||||
if type(exc) in (psycopg.DatabaseError, psycopg.OperationalError):
|
||||
self.close()
|
||||
else:
|
||||
raise exc
|
||||
raise PostgresConnectionException('connection problems') from exc
|
||||
|
||||
def close(self, silent: bool = False) -> bool:
|
||||
"""Close the psycopg connection to postgres.
|
||||
|
||||
:param silent: whether the method should not write logs.
|
||||
|
||||
:returns: ``True`` if ``psycopg`` connection was closed, ``False`` otherwise.``
|
||||
"""
|
||||
ret = False
|
||||
if self._connection and self._connection.closed == 0:
|
||||
self._connection.close()
|
||||
if not silent:
|
||||
logger.info("closed patroni %s connection to postgres", self._name)
|
||||
ret = True
|
||||
self._connection = None
|
||||
return ret
|
||||
|
||||
|
||||
class ConnectionPool:
|
||||
"""Helper class to manage named connections from Patroni to PostgreSQL.
|
||||
|
||||
The instance keeps named :class:`NamedConnection` objects and parameters that must be used for new connections.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Create an instance of :class:`ConnectionPool` class."""
|
||||
self._lock = Lock()
|
||||
self._connections: Dict[str, NamedConnection] = {}
|
||||
self._conn_kwargs: Dict[str, Any] = {}
|
||||
|
||||
@property
|
||||
def conn_kwargs(self) -> Dict[str, Any]:
|
||||
"""Connection parameters that must be used for new ``psycopg`` connections."""
|
||||
with self._lock:
|
||||
return self._conn_kwargs.copy()
|
||||
|
||||
@conn_kwargs.setter
|
||||
def conn_kwargs(self, value: Dict[str, Any]) -> None:
|
||||
"""Set new connection parameters.
|
||||
|
||||
:param value: :class:`dict` object with connection parameters.
|
||||
"""
|
||||
with self._lock:
|
||||
self._conn_kwargs = value
|
||||
|
||||
def get(self, name: str, kwargs_override: Optional[Dict[str, Any]] = None) -> NamedConnection:
|
||||
"""Get a new named :class:`NamedConnection` object from the pool.
|
||||
|
||||
.. note::
|
||||
Creates a new :class:`NamedConnection` object if it doesn't yet exist in the pool.
|
||||
|
||||
:param name: name of the connection.
|
||||
:param kwargs_override: :class:`dict` object with connection parameters that should be
|
||||
different from default values provided by :attr:`conn_kwargs`.
|
||||
|
||||
:returns: :class:`NamedConnection` object.
|
||||
"""
|
||||
with self._lock:
|
||||
if name not in self._connections:
|
||||
self._connections[name] = NamedConnection(self, name, kwargs_override)
|
||||
return self._connections[name]
|
||||
def cursor(self) -> Union['cursor', 'Cursor[Any]']:
|
||||
if not self._cursor_holder or self._cursor_holder.closed or self._cursor_holder.connection.closed != 0:
|
||||
logger.info("establishing a new patroni connection to the postgres cluster")
|
||||
self._cursor_holder = self.get().cursor()
|
||||
return self._cursor_holder
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close all named connections from Patroni to PostgreSQL registered in the pool."""
|
||||
with self._lock:
|
||||
if any(conn.close(True) for conn in self._connections.values()):
|
||||
logger.info("closed patroni connections to postgres")
|
||||
if self._connection and self._connection.closed == 0:
|
||||
self._connection.close()
|
||||
logger.info("closed patroni connection to the postgresql cluster")
|
||||
self._cursor_holder = self._connection = None
|
||||
|
||||
|
||||
@contextmanager
|
||||
|
||||
@@ -158,7 +158,7 @@ class Rewind(object):
|
||||
def _get_local_timeline_lsn(self) -> Tuple[Optional[bool], Optional[int], Optional[int]]:
|
||||
if self._postgresql.is_running(): # if postgres is running - get timeline from replication connection
|
||||
in_recovery = True
|
||||
timeline = self._postgresql.received_timeline() or self._postgresql.get_replica_timeline()
|
||||
timeline = self._postgresql.get_replica_timeline()
|
||||
lsn = self._postgresql.replayed_location()
|
||||
else: # otherwise analyze pg_controldata output
|
||||
in_recovery, timeline, lsn = self._get_local_timeline_lsn_from_controldata()
|
||||
@@ -280,7 +280,7 @@ class Rewind(object):
|
||||
"""After promote issue a CHECKPOINT from a new thread and asynchronously check the result.
|
||||
In case if CHECKPOINT failed, just check that timeline in pg_control was updated."""
|
||||
|
||||
if self._state != REWIND_STATUS.CHECKPOINT and self._postgresql.is_primary():
|
||||
if self._state != REWIND_STATUS.CHECKPOINT and self._postgresql.is_leader():
|
||||
with self._checkpoint_task_lock:
|
||||
if self._checkpoint_task:
|
||||
with self._checkpoint_task:
|
||||
|
||||
+33
-24
@@ -186,7 +186,7 @@ class SlotsHandler:
|
||||
self.pg_replslot_dir = os.path.join(self._postgresql.data_dir, 'pg_replslot')
|
||||
self.schedule()
|
||||
|
||||
def _query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]:
|
||||
def _query(self, sql: str, *params: Any) -> Union['cursor', 'Cursor[Any]']:
|
||||
"""Helper method for :meth:`Postgresql.query`.
|
||||
|
||||
:param sql: SQL statement to execute.
|
||||
@@ -263,8 +263,9 @@ class SlotsHandler:
|
||||
extra = ", catalog_xmin, pg_catalog.pg_wal_lsn_diff(confirmed_flush_lsn, '0/0')::bigint" \
|
||||
if self._postgresql.major_version >= 100000 else ""
|
||||
skip_temp_slots = ' WHERE NOT temporary' if self._postgresql.major_version >= 100000 else ''
|
||||
for r in self._query('SELECT slot_name, slot_type, plugin, database, datoid'
|
||||
f'{extra} FROM pg_catalog.pg_replication_slots{skip_temp_slots}'):
|
||||
cursor = self._query(f'SELECT slot_name, slot_type, plugin, database, datoid'
|
||||
f'{extra} FROM pg_catalog.pg_replication_slots{skip_temp_slots}')
|
||||
for r in cursor:
|
||||
value = {'type': r[1]}
|
||||
if r[1] == 'logical':
|
||||
value.update(plugin=r[2], database=r[3], datoid=r[4])
|
||||
@@ -307,13 +308,16 @@ class SlotsHandler:
|
||||
``dropped`` is ``True`` if the slot was successfully dropped. If the slot was not found return
|
||||
``False`` for both.
|
||||
"""
|
||||
rows = self._query(('WITH slots AS (SELECT slot_name, active'
|
||||
' FROM pg_catalog.pg_replication_slots WHERE slot_name = %s),'
|
||||
' dropped AS (SELECT pg_catalog.pg_drop_replication_slot(slot_name),'
|
||||
' true AS dropped FROM slots WHERE not active) '
|
||||
'SELECT active, COALESCE(dropped, false) FROM slots'
|
||||
' FULL OUTER JOIN dropped ON true'), name)
|
||||
return rows[0] if rows else (False, False)
|
||||
cursor = self._query(('WITH slots AS (SELECT slot_name, active'
|
||||
' FROM pg_catalog.pg_replication_slots WHERE slot_name = %s),'
|
||||
' dropped AS (SELECT pg_catalog.pg_drop_replication_slot(slot_name),'
|
||||
' true AS dropped FROM slots WHERE not active) '
|
||||
'SELECT active, COALESCE(dropped, false) FROM slots'
|
||||
' FULL OUTER JOIN dropped ON true'), name)
|
||||
row = cursor.fetchone()
|
||||
if not row:
|
||||
row = (False, False)
|
||||
return row[0], row[1]
|
||||
|
||||
def _drop_incorrect_slots(self, cluster: Cluster, slots: Dict[str, Any], paused: bool) -> None:
|
||||
"""Compare required slots and configured as permanent slots with those found, dropping extraneous ones.
|
||||
@@ -384,7 +388,8 @@ class SlotsHandler:
|
||||
|
||||
:yields: connection cursor object, note implementation varies depending on version of :mod:`psycopg`.
|
||||
"""
|
||||
conn_kwargs = {**self._postgresql.connection_pool.conn_kwargs, **kwargs}
|
||||
conn_kwargs = self._postgresql.config.local_connect_kwargs
|
||||
conn_kwargs.update(kwargs)
|
||||
with get_connection_cursor(**conn_kwargs) as cur:
|
||||
yield cur
|
||||
|
||||
@@ -434,7 +439,7 @@ class SlotsHandler:
|
||||
self._advance = SlotsAdvanceThread(self)
|
||||
return self._advance.schedule(slots)
|
||||
|
||||
def _ensure_logical_slots_replica(self, slots: Dict[str, Any]) -> List[str]:
|
||||
def _ensure_logical_slots_replica(self, cluster: Cluster, slots: Dict[str, Any]) -> List[str]:
|
||||
"""Update logical *slots* on replicas.
|
||||
|
||||
If the logical slot already exists, copy state information into the replication slots structure stored in the
|
||||
@@ -444,6 +449,7 @@ class SlotsHandler:
|
||||
As logical slots can only be created when the primary is available, pass the list of slots that need to be
|
||||
copied back to the caller. They will be created on replicas with :meth:`SlotsHandler.copy_logical_slots`.
|
||||
|
||||
:param cluster: object containing stateful information for the cluster.
|
||||
:param slots: A dictionary mapping slot name to slot attributes. This method only considers a slot
|
||||
if the value is a dictionary with the key ``type`` and a value of ``logical``.
|
||||
|
||||
@@ -458,16 +464,15 @@ class SlotsHandler:
|
||||
continue
|
||||
|
||||
# If the logical already exists, copy some information about it into the original structure
|
||||
if name in self._replication_slots and compare_slots(value, self._replication_slots[name]):
|
||||
if self._replication_slots.get(name, {}).get('datoid'):
|
||||
self._copy_items(self._replication_slots[name], value)
|
||||
if 'lsn' in value: # The slot has feedback in DCS
|
||||
if cluster.slots and name in cluster.slots:
|
||||
try: # Skip slots that don't need to be advanced
|
||||
if value['confirmed_flush_lsn'] < int(value['lsn']):
|
||||
advance_slots[value['database']][name] = int(value['lsn'])
|
||||
if value['confirmed_flush_lsn'] < int(cluster.slots[name]):
|
||||
advance_slots[value['database']][name] = int(cluster.slots[name])
|
||||
except Exception as e:
|
||||
logger.error('Failed to parse "%s": %r', value['lsn'], e)
|
||||
elif name not in self._replication_slots and 'lsn' in value:
|
||||
# We want to copy only slots with feedback in a DCS
|
||||
logger.error('Failed to parse "%s": %r', cluster.slots[name], e)
|
||||
elif cluster.slots and name in cluster.slots: # We want to copy only slots with feedback in a DCS
|
||||
create_slots.append(name)
|
||||
|
||||
# Slots to be copied from the primary should be removed from the *slots* structure,
|
||||
@@ -509,12 +514,13 @@ class SlotsHandler:
|
||||
|
||||
self._ensure_physical_slots(slots)
|
||||
|
||||
if self._postgresql.is_primary():
|
||||
if self._postgresql.is_leader():
|
||||
self._logical_slots_processing_queue.clear()
|
||||
self._ensure_logical_slots_primary(slots)
|
||||
else:
|
||||
elif cluster.slots and slots:
|
||||
self.check_logical_slots_readiness(cluster, replicatefrom)
|
||||
ret = self._ensure_logical_slots_replica(slots)
|
||||
|
||||
ret = self._ensure_logical_slots_replica(cluster, slots)
|
||||
|
||||
self._replication_slots = slots
|
||||
except Exception:
|
||||
@@ -595,8 +601,11 @@ class SlotsHandler:
|
||||
|
||||
# Replica isn't streaming or the hot_standby_feedback isn't enabled
|
||||
try:
|
||||
if not self._query("SELECT pg_catalog.current_setting('hot_standby_feedback')::boolean")[0][0]:
|
||||
logger.error('Logical slot failover requires "hot_standby_feedback". Please check postgresql.auto.conf')
|
||||
cur = self._query("SELECT pg_catalog.current_setting('hot_standby_feedback')::boolean")
|
||||
row = cur.fetchone()
|
||||
if row and not row[0]:
|
||||
logger.error('Logical slot failover requires "hot_standby_feedback".'
|
||||
' Please check postgresql.auto.conf')
|
||||
except Exception as e:
|
||||
logger.error('Failed to check the hot_standby_feedback setting: %r', e)
|
||||
return False
|
||||
|
||||
@@ -182,7 +182,7 @@ class _ReplicaList(List[_Replica]):
|
||||
swapping, but only if lag on this member is exceeding a threshold (``maximum_lag_on_syncnode``).
|
||||
|
||||
:ivar max_lsn: maximum value of ``_Replica.lsn`` among all values. In case if there is just one
|
||||
element in the list we take value of ``pg_current_wal_flush_lsn()``.
|
||||
element in the list we take value of ``pg_current_wal_lsn()``.
|
||||
"""
|
||||
|
||||
def __init__(self, postgresql: 'Postgresql', cluster: Cluster) -> None:
|
||||
@@ -209,7 +209,7 @@ class _ReplicaList(List[_Replica]):
|
||||
# 2. can be mapped to a ``Member`` of the ``Cluster``:
|
||||
# a. ``Member`` doesn't have ``nosync`` tag set;
|
||||
# b. PostgreSQL on the member is known to be running and accepting client connections.
|
||||
if member and row[sort_col] is not None and member.is_running and not member.nosync:
|
||||
if member and row[sort_col] is not None and member.is_running and not member.tags.get('nosync', False):
|
||||
self.append(_Replica(row['pid'], row['application_name'],
|
||||
row['sync_state'], row[sort_col], bool(member.nofailover)))
|
||||
|
||||
@@ -339,7 +339,7 @@ END;$$""")
|
||||
sync_param = next(iter(sync), None)
|
||||
|
||||
if not (self._postgresql.config.set_synchronous_standby_names(sync_param)
|
||||
and self._postgresql.state == 'running' and self._postgresql.is_primary()) or has_asterisk:
|
||||
and self._postgresql.state == 'running' and self._postgresql.is_leader()) or has_asterisk:
|
||||
return
|
||||
|
||||
time.sleep(0.1) # Usualy it takes 1ms to reload postgresql.conf, but we will give it 100ms
|
||||
|
||||
@@ -290,7 +290,7 @@ def _load_postgres_gucs_validators() -> None:
|
||||
Any problem faced while reading or parsing files will be logged as a ``WARNING`` by the child function, and the
|
||||
corresponding file or validator will be ignored.
|
||||
|
||||
By default, Patroni only ships the file ``0_postgres.yml``, which contains Community Postgres GUCs validators, but
|
||||
By default Patroni only ships the file ``0_postgres.yml``, which contains Community Postgres GUCs validators, but
|
||||
that behavior can be extended. For example: if a vendor wants to add GUC validators to Patroni for covering a custom
|
||||
Postgres build, then they can create their custom YAML files under ``available_parameters`` directory.
|
||||
|
||||
@@ -300,10 +300,8 @@ def _load_postgres_gucs_validators() -> None:
|
||||
writes them to ``postgresql.conf`` if running PG 12 and above).
|
||||
|
||||
Then, each of these sections, if specified, may contain one or more attributes with the following structure:
|
||||
|
||||
* key: the name of a GUC;
|
||||
* value: a list of validators. Each item in the list must contain a ``type`` attribute, which must be one among:
|
||||
|
||||
* ``Bool``; or
|
||||
* ``Integer``; or
|
||||
* ``Real``; or
|
||||
@@ -315,7 +313,6 @@ def _load_postgres_gucs_validators() -> None:
|
||||
class in this module.
|
||||
|
||||
.. seealso::
|
||||
|
||||
* :class:`Bool`;
|
||||
* :class:`Integer`;
|
||||
* :class:`Real`;
|
||||
@@ -328,62 +325,61 @@ def _load_postgres_gucs_validators() -> None:
|
||||
This is a sample content for an YAML file based on Postgres GUCs, showing each of the supported types and
|
||||
sections:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
parameters:
|
||||
archive_command:
|
||||
- type: String
|
||||
version_from: 90300
|
||||
version_till: null
|
||||
archive_mode:
|
||||
- type: Bool
|
||||
version_from: 90300
|
||||
version_till: 90500
|
||||
- type: EnumBool
|
||||
version_from: 90500
|
||||
version_till: null
|
||||
possible_values:
|
||||
- always
|
||||
archive_timeout:
|
||||
- type: Integer
|
||||
version_from: 90300
|
||||
version_till: null
|
||||
min_val: 0
|
||||
max_val: 1073741823
|
||||
unit: s
|
||||
autovacuum_vacuum_cost_delay:
|
||||
- type: Integer
|
||||
version_from: 90300
|
||||
version_till: 120000
|
||||
min_val: -1
|
||||
max_val: 100
|
||||
unit: ms
|
||||
- type: Real
|
||||
version_from: 120000
|
||||
version_till: null
|
||||
min_val: -1
|
||||
max_val: 100
|
||||
unit: ms
|
||||
client_min_messages:
|
||||
- type: Enum
|
||||
version_from: 90300
|
||||
version_till: null
|
||||
possible_values:
|
||||
- debug5
|
||||
- debug4
|
||||
- debug3
|
||||
- debug2
|
||||
- debug1
|
||||
- log
|
||||
- notice
|
||||
- warning
|
||||
- error
|
||||
recovery_parameters:
|
||||
archive_cleanup_command:
|
||||
- type: String
|
||||
version_from: 90300
|
||||
version_till: null
|
||||
|
||||
```yaml
|
||||
parameters:
|
||||
archive_command:
|
||||
- type: String
|
||||
version_from: 90300
|
||||
version_till: null
|
||||
archive_mode:
|
||||
- type: Bool
|
||||
version_from: 90300
|
||||
version_till: 90500
|
||||
- type: EnumBool
|
||||
version_from: 90500
|
||||
version_till: null
|
||||
possible_values:
|
||||
- always
|
||||
archive_timeout:
|
||||
- type: Integer
|
||||
version_from: 90300
|
||||
version_till: null
|
||||
min_val: 0
|
||||
max_val: 1073741823
|
||||
unit: s
|
||||
autovacuum_vacuum_cost_delay:
|
||||
- type: Integer
|
||||
version_from: 90300
|
||||
version_till: 120000
|
||||
min_val: -1
|
||||
max_val: 100
|
||||
unit: ms
|
||||
- type: Real
|
||||
version_from: 120000
|
||||
version_till: null
|
||||
min_val: -1
|
||||
max_val: 100
|
||||
unit: ms
|
||||
client_min_messages:
|
||||
- type: Enum
|
||||
version_from: 90300
|
||||
version_till: null
|
||||
possible_values:
|
||||
- debug5
|
||||
- debug4
|
||||
- debug3
|
||||
- debug2
|
||||
- debug1
|
||||
- log
|
||||
- notice
|
||||
- warning
|
||||
- error
|
||||
recovery_parameters:
|
||||
archive_cleanup_command:
|
||||
- type: String
|
||||
version_from: 90300
|
||||
version_till: null
|
||||
```
|
||||
"""
|
||||
conf_dir = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)),
|
||||
@@ -438,15 +434,13 @@ def _transform_parameter_value(validators: MutableMapping[str, Tuple[_Transforma
|
||||
:param value: value of the Postgres GUC.
|
||||
:param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres
|
||||
GUC. Used for a couple purposes:
|
||||
|
||||
* Disallow writing GUCs to ``postgresql.conf`` (or ``recovery.conf``) that does not exist in Postgres *version*;
|
||||
* Avoid ignoring GUC *name* if it does not have a validator in *validators*, but is a valid GUC in Postgres
|
||||
*version*.
|
||||
*version*.
|
||||
|
||||
:returns: the return value may be one among:
|
||||
|
||||
* *value* transformed to the expected format for GUC *name* in Postgres *version*, if *name* is present
|
||||
in *available_gucs* and has a validator in *validators* for the corresponding Postgres *version*; or
|
||||
* *value* transformed to the expected format for GUC *name* in Postgres *version*, if *name* is present in
|
||||
*available_gucs* and has a validator in *validators* for the corresponding Postgres *version*; or
|
||||
* The own *value* if *name* is present in *available_gucs* but not in *validators*; or
|
||||
* ``None`` if *name* is not present in *available_gucs*.
|
||||
"""
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
"""Tags handling."""
|
||||
import abc
|
||||
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
|
||||
class Tags(abc.ABC):
|
||||
"""An abstract class that encapsulates all the ``tags`` logic.
|
||||
|
||||
Child classes that want to use provided facilities must implement ``tags`` abstract property.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _filter_tags(tags: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Get tags configured for this node, if any.
|
||||
|
||||
Handle both predefined Patroni tags and custom defined tags.
|
||||
|
||||
.. note::
|
||||
A custom tag is any tag added to the configuration ``tags`` section that is not one of ``clonefrom``,
|
||||
``nofailover``, ``noloadbalance`` or ``nosync``.
|
||||
|
||||
For the Patroni predefined tags, the returning object will only contain them if they are enabled as they
|
||||
all are boolean values that default to disabled.
|
||||
|
||||
:returns: a dictionary of tags set for this node. The key is the tag name, and the value is the corresponding
|
||||
tag value.
|
||||
"""
|
||||
return {tag: value for tag, value in tags.items()
|
||||
if tag not in ('clonefrom', 'nofailover', 'noloadbalance', 'nosync') or value}
|
||||
|
||||
@property
|
||||
@abc.abstractmethod
|
||||
def tags(self) -> Dict[str, Any]:
|
||||
"""Configured tags.
|
||||
|
||||
Must be implemented in a child class.
|
||||
"""
|
||||
raise NotImplementedError # pragma: no cover
|
||||
|
||||
@property
|
||||
def clonefrom(self) -> bool:
|
||||
"""``True`` if ``clonefrom`` tag is ``True``, else ``False``."""
|
||||
return self.tags.get('clonefrom', False)
|
||||
|
||||
@property
|
||||
def nofailover(self) -> bool:
|
||||
"""``True`` if ``nofailover`` is ``True``, else ``False``."""
|
||||
return bool(self.tags.get('nofailover', False))
|
||||
|
||||
@property
|
||||
def noloadbalance(self) -> bool:
|
||||
"""``True`` if ``noloadbalance`` is ``True``, else ``False``."""
|
||||
return bool(self.tags.get('noloadbalance', False))
|
||||
|
||||
@property
|
||||
def nosync(self) -> bool:
|
||||
"""``True`` if ``nosync`` is ``True``, else ``False``."""
|
||||
return bool(self.tags.get('nosync', False))
|
||||
|
||||
@property
|
||||
def replicatefrom(self) -> Optional[str]:
|
||||
"""Value of ``replicatefrom`` tag, if any."""
|
||||
return self.tags.get('replicatefrom')
|
||||
+2
-72
@@ -9,8 +9,6 @@
|
||||
:var DBL_RE: regular expression to match double precision numbers, signed or unsigned. Matches scientific notation too.
|
||||
:var WHITESPACE_RE: regular expression to match whitespace characters
|
||||
"""
|
||||
import datetime
|
||||
import dateutil.parser
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
@@ -18,11 +16,9 @@ import platform
|
||||
import random
|
||||
import re
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from enum import Enum
|
||||
from shlex import split
|
||||
|
||||
from typing import Any, Callable, Dict, Iterator, List, Optional, Union, Tuple, Type, TYPE_CHECKING
|
||||
@@ -474,18 +470,6 @@ def _sleep(interval: Union[int, float]) -> None:
|
||||
time.sleep(interval)
|
||||
|
||||
|
||||
def read_stripped(file_path: str) -> Iterator[str]:
|
||||
"""Iterate over stripped lines in the given file.
|
||||
|
||||
:param file_path: path to the file to read from
|
||||
|
||||
:yields: each line from the given file stripped
|
||||
"""
|
||||
with open(file_path) as f:
|
||||
for line in f:
|
||||
yield line.strip()
|
||||
|
||||
|
||||
class RetryFailedError(PatroniException):
|
||||
"""Maximum number of attempts exhausted in retry operation."""
|
||||
|
||||
@@ -839,9 +823,8 @@ def cluster_as_json(cluster: 'Cluster', global_config: Optional['GlobalConfig']
|
||||
ret['pause'] = True
|
||||
if cluster.failover and cluster.failover.scheduled_at:
|
||||
ret['scheduled_switchover'] = {'at': cluster.failover.scheduled_at.isoformat()}
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
assert cluster.failover.leader
|
||||
ret['scheduled_switchover']['from'] = cluster.failover.leader
|
||||
if cluster.failover.leader:
|
||||
ret['scheduled_switchover']['from'] = cluster.failover.leader
|
||||
if cluster.failover.candidate:
|
||||
ret['scheduled_switchover']['to'] = cluster.failover.candidate
|
||||
return ret
|
||||
@@ -1032,56 +1015,3 @@ def unquote(string: str) -> str:
|
||||
except ValueError:
|
||||
ret = string
|
||||
return ret
|
||||
|
||||
|
||||
def get_major_version(bin_dir: Optional[str] = None, bin_name: str = 'postgres') -> str:
|
||||
"""Get the major version of PostgreSQL.
|
||||
|
||||
It is based on the output of ``postgres --version``.
|
||||
|
||||
:param bin_dir: path to the PostgreSQL binaries directory. If ``None`` or an empty string, it will use the first
|
||||
*bin_name* binary that is found by the subprocess in the ``PATH``.
|
||||
:param bin_name: name of the postgres binary to call (``postgres`` by default)
|
||||
|
||||
:returns: the PostgreSQL major version.
|
||||
|
||||
:raises:
|
||||
:exc:`~patroni.exceptions.PatroniException`: if the postgres binary call failed due to :exc:`OSError`.
|
||||
|
||||
:Example:
|
||||
|
||||
* Returns `9.6` for PostgreSQL 9.6.24
|
||||
* Returns `15` for PostgreSQL 15.2
|
||||
"""
|
||||
if not bin_dir:
|
||||
binary = bin_name
|
||||
else:
|
||||
binary = os.path.join(bin_dir, bin_name)
|
||||
try:
|
||||
version = subprocess.check_output([binary, '--version']).decode()
|
||||
except OSError as e:
|
||||
raise PatroniException(f'Failed to get postgres version: {e}')
|
||||
version = re.match(r'^[^\s]+ [^\s]+ (\d+)(\.(\d+))?', version)
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
assert version is not None
|
||||
return '.'.join([version.group(1), version.group(3)]) if int(version.group(1)) < 10 else version.group(1)
|
||||
|
||||
|
||||
class ParseScheduleErrors(Enum):
|
||||
NO_TIMEZONE = ('Timezone information is mandatory for the scheduled {action}', 400)
|
||||
SCHEDULED_IN_PAST = ('Cannot schedule {action} in the past', 422)
|
||||
PARSING_ERROR = ('Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601', 422)
|
||||
|
||||
|
||||
def parse_schedule(schedule: Optional[str]) -> Tuple[Optional[ParseScheduleErrors], Optional[datetime.datetime]]:
|
||||
scheduled_at = None
|
||||
if schedule is not None:
|
||||
try:
|
||||
scheduled_at = dateutil.parser.parse(schedule)
|
||||
if scheduled_at.tzinfo is None:
|
||||
return ParseScheduleErrors.NO_TIMEZONE, scheduled_at
|
||||
elif scheduled_at < datetime.datetime.now(tzutc):
|
||||
return ParseScheduleErrors.SCHEDULED_IN_PAST, scheduled_at
|
||||
except (ValueError, TypeError):
|
||||
return ParseScheduleErrors.PARSING_ERROR, scheduled_at
|
||||
return None, scheduled_at
|
||||
|
||||
+67
-47
@@ -6,16 +6,17 @@ This module contains facilities for validating configuration of Patroni processe
|
||||
:var schema: configuration schema of the daemon launched by ``patroni`` command.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
|
||||
from typing import Any, Dict, Union, Iterator, List, Optional as OptionalType, Tuple
|
||||
from typing import Any, Dict, Union, Iterator, List, Optional as OptionalType, Tuple, TYPE_CHECKING
|
||||
|
||||
from .collections import CaseInsensitiveSet
|
||||
|
||||
from .dcs import dcs_modules
|
||||
from .exceptions import ConfigParseError
|
||||
from .utils import parse_int, split_host_port, data_directory_is_empty, get_major_version
|
||||
from .utils import parse_int, split_host_port, data_directory_is_empty
|
||||
|
||||
|
||||
def data_directory_empty(data_dir: str) -> bool:
|
||||
@@ -203,6 +204,31 @@ def get_bin_name(bin_name: str) -> str:
|
||||
return (schema.data.get('postgresql', {}).get('bin_name', {}) or {}).get(bin_name, bin_name)
|
||||
|
||||
|
||||
def get_major_version(bin_dir: OptionalType[str] = None) -> str:
|
||||
"""Get the major version of PostgreSQL.
|
||||
|
||||
It is based on the output of ``postgres --version``.
|
||||
|
||||
:param bin_dir: path to PostgreSQL binaries directory. If ``None`` it will use the first ``postgres`` binary that
|
||||
is found by subprocess in the ``PATH``.
|
||||
:returns: the PostgreSQL major version.
|
||||
|
||||
:Example:
|
||||
|
||||
* Returns `9.6` for PostgreSQL 9.6.24
|
||||
* Returns `15` for PostgreSQL 15.2
|
||||
"""
|
||||
if not bin_dir:
|
||||
binary = get_bin_name('postgres')
|
||||
else:
|
||||
binary = os.path.join(bin_dir, get_bin_name('postgres'))
|
||||
version = subprocess.check_output([binary, '--version']).decode()
|
||||
version = re.match(r'^[^\s]+ [^\s]+ (\d+)(\.(\d+))?', version)
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
assert version is not None
|
||||
return '.'.join([version.group(1), version.group(3)]) if int(version.group(1)) < 10 else version.group(1)
|
||||
|
||||
|
||||
def validate_data_dir(data_dir: str) -> bool:
|
||||
"""Validate the value of ``postgresql.data_dir`` configuration option.
|
||||
|
||||
@@ -240,7 +266,7 @@ def validate_data_dir(data_dir: str) -> bool:
|
||||
raise ConfigParseError("data dir for the cluster is not empty, but doesn't contain"
|
||||
" \"{}\" directory".format(waldir))
|
||||
bin_dir = schema.data.get("postgresql", {}).get("bin_dir", None)
|
||||
major_version = get_major_version(bin_dir, get_bin_name('postgres'))
|
||||
major_version = get_major_version(bin_dir)
|
||||
if pgversion != major_version:
|
||||
raise ConfigParseError("data_dir directory postgresql version ({}) doesn't match with "
|
||||
"'postgres --version' output ({})".format(pgversion, major_version))
|
||||
@@ -333,17 +359,15 @@ class Case(object):
|
||||
"""Create a :class:`Case` object.
|
||||
|
||||
:param schema: the schema for validating a set of attributes that may be available in the configuration.
|
||||
Each key is the configuration that is available in a given scope and that should be validated,
|
||||
and the related value is the validation function or expected type.
|
||||
Each key is the configuration that is available in a given scope and that should be validated, and the
|
||||
related value is the validation function or expected type.
|
||||
|
||||
:Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
Case({
|
||||
"host": validate_host_port,
|
||||
"url": str,
|
||||
})
|
||||
Case({
|
||||
"host": validate_host_port,
|
||||
"url": str,
|
||||
})
|
||||
|
||||
That will check that ``host`` configuration, if given, is valid based on :func:`validate_host_port`, and will
|
||||
also check that ``url`` configuration, if given, is a ``str`` instance.
|
||||
@@ -365,16 +389,14 @@ class Or(object):
|
||||
|
||||
:Example:
|
||||
|
||||
.. code-block:: python
|
||||
Or("host", "hosts"): Case({
|
||||
"host": validate_host_port,
|
||||
"hosts": Or(comma_separated_host_port, [validate_host_port]),
|
||||
})
|
||||
|
||||
Or("host", "hosts"): Case({
|
||||
"host": validate_host_port,
|
||||
"hosts": Or(comma_separated_host_port, [validate_host_port]),
|
||||
})
|
||||
|
||||
The outer :class:`Or` is used to define that ``host`` and ``hosts`` are possible options in this scope.
|
||||
The inner :class`Or` in the ``hosts`` key value is used to define that ``hosts`` option is valid if either
|
||||
of :func:`comma_separated_host_port` or :func:`validate_host_port` succeed to validate it.
|
||||
The outer :class:`Or` is used to define that ``host`` and ``hosts`` are possible options in this scope.
|
||||
The inner :class`Or` in the ``hosts`` key value is used to define that ``hosts`` option is valid if either of
|
||||
:func:`comma_separated_host_port` or :func:`validate_host_port` succeed to validate it.
|
||||
"""
|
||||
self.args = args
|
||||
|
||||
@@ -539,34 +561,32 @@ class Schema(object):
|
||||
|
||||
:Example:
|
||||
|
||||
.. code-block:: python
|
||||
Schema({
|
||||
"application_name": str,
|
||||
"bind": {
|
||||
"host": validate_host,
|
||||
"port": int,
|
||||
},
|
||||
"aliases": [str],
|
||||
Optional("data_directory"): "/var/lib/myapp",
|
||||
Or("log_to_file", "log_to_db"): Case({
|
||||
"log_to_file": bool,
|
||||
"log_to_db": bool,
|
||||
}),
|
||||
"version": Or(int, float),
|
||||
})
|
||||
|
||||
Schema({
|
||||
"application_name": str,
|
||||
"bind": {
|
||||
"host": validate_host,
|
||||
"port": int,
|
||||
},
|
||||
"aliases": [str],
|
||||
Optional("data_directory"): "/var/lib/myapp",
|
||||
Or("log_to_file", "log_to_db"): Case({
|
||||
"log_to_file": bool,
|
||||
"log_to_db": bool,
|
||||
}),
|
||||
"version": Or(int, float),
|
||||
})
|
||||
This sample schema defines that your YAML configuration follows these rules:
|
||||
|
||||
This sample schema defines that your YAML configuration follows these rules:
|
||||
|
||||
* It must contain an ``application_name`` entry which value should be a :class:`str` instance;
|
||||
* It must contain a ``bind.host`` entry which value should be valid as per function ``validate_host``;
|
||||
* It must contain a ``bind.port`` entry which value should be an :class:`int` instance;
|
||||
* It must contain a ``aliases`` entry which value should be a :class:`list` of :class:`str` instances;
|
||||
* It may optionally contain a ``data_directory`` entry, with a value which should be a string;
|
||||
* It must contain at least one of ``log_to_file`` or ``log_to_db``, with a value which should be a
|
||||
:class:`bool` instance;
|
||||
* It must contain a ``version`` entry which value should be either an :class:`int` or a :class:`float`
|
||||
instance.
|
||||
* It must contain an ``application_name`` entry which value should be a :class:`str` instance;
|
||||
* It must contain a ``bind.host`` entry which value should be valid as per function ``validate_host``;
|
||||
* It must contain a ``bind.port`` entry which value should be an :class:`int` instance;
|
||||
* It must contain a ``aliases`` entry which value should be a :class:`list` of :class:`str` instances;
|
||||
* It may optionally contain a ``data_directory`` entry, with a value which should be a string;
|
||||
* It must contain at least one of ``log_to_file`` or ``log_to_db``, with a value which should be a
|
||||
:class:`bool` instance;
|
||||
* It must contain a ``version`` entry which value should be either an :class:`int` or a :class:`float`
|
||||
instance.
|
||||
"""
|
||||
self.validator = validator
|
||||
|
||||
|
||||
+1
-1
@@ -2,4 +2,4 @@
|
||||
|
||||
:var __version__: the current Patroni version.
|
||||
"""
|
||||
__version__ = '3.1.0'
|
||||
__version__ = '3.1.1'
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
sphinx>=4
|
||||
sphinx_rtd_theme>1
|
||||
sphinxcontrib-apidoc
|
||||
sphinx-github-style<1.0.3
|
||||
sphinx-github-style
|
||||
pyyaml
|
||||
|
||||
+6
-30
@@ -104,9 +104,9 @@ class MockCursor(object):
|
||||
elif sql.startswith('SELECT slot_name, slot_type, datname, plugin, catalog_xmin'):
|
||||
self.results = [('ls', 'logical', 'a', 'b', 100, 500, b'123456')]
|
||||
elif sql.startswith('SELECT slot_name'):
|
||||
self.results = [('blabla', 'physical'), ('foobar', 'physical'), ('ls', 'logical', 'b', 'a', 5, 100, 500)]
|
||||
self.results = [('blabla', 'physical'), ('foobar', 'physical'), ('ls', 'logical', 'a', 'b', 5, 100, 500)]
|
||||
elif sql.startswith('WITH slots AS (SELECT slot_name, active'):
|
||||
self.results = [(False, True)] if self.rowcount == 1 else []
|
||||
self.results = [(False, True)] if self.rowcount == 1 else [None]
|
||||
elif sql.startswith('SELECT CASE WHEN pg_catalog.pg_is_in_recovery()'):
|
||||
self.results = [(1, 2, 1, 0, False, 1, 1, None, None, 'streaming', '',
|
||||
[{"slot_name": "ls", "confirmed_flush_lsn": 12345}],
|
||||
@@ -114,22 +114,10 @@ class MockCursor(object):
|
||||
elif sql.startswith('SELECT pg_catalog.pg_is_in_recovery()'):
|
||||
self.results = [(False, 2)]
|
||||
elif sql.startswith('SELECT pg_catalog.pg_postmaster_start_time'):
|
||||
self.results = [(datetime.datetime.now(tzutc),)]
|
||||
elif sql.startswith('SELECT name, current_setting(name) FROM pg_settings'):
|
||||
self.results = [('data_directory', 'data'),
|
||||
('hba_file', os.path.join('data', 'pg_hba.conf')),
|
||||
('ident_file', os.path.join('data', 'pg_ident.conf')),
|
||||
('max_connections', 42),
|
||||
('max_locks_per_transaction', 73),
|
||||
('max_prepared_transactions', 0),
|
||||
('max_replication_slots', 21),
|
||||
('max_wal_senders', 37),
|
||||
('track_commit_timestamp', 'off'),
|
||||
('wal_level', 'replica'),
|
||||
('listen_addresses', '6.6.6.6'),
|
||||
('port', 1984),
|
||||
('archive_command', 'my archive command'),
|
||||
('cluster_name', 'my_cluster')]
|
||||
replication_info = '[{"application_name":"walreceiver","client_addr":"1.2.3.4",' +\
|
||||
'"state":"streaming","sync_state":"async","sync_priority":0}]'
|
||||
now = datetime.datetime.now(tzutc)
|
||||
self.results = [(now, 0, '', 0, '', False, now, 'streaming', None, replication_info)]
|
||||
elif sql.startswith('SELECT name, setting'):
|
||||
self.results = [('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
|
||||
('wal_block_size', '8192', None, 'integer', 'internal'),
|
||||
@@ -140,8 +128,6 @@ class MockCursor(object):
|
||||
('listen_addresses', '*', None, 'string', 'postmaster'),
|
||||
('autovacuum', 'on', None, 'bool', 'sighup'),
|
||||
('unix_socket_directories', '/tmp', None, 'string', 'postmaster')]
|
||||
elif sql.startswith('SELECT COUNT(*) FROM pg_catalog.pg_settings'):
|
||||
self.results = [(1,)]
|
||||
elif sql.startswith('IDENTIFY_SYSTEM'):
|
||||
self.results = [('1', 3, '0/402EEC0', '')]
|
||||
elif sql.startswith('TIMELINE_HISTORY '):
|
||||
@@ -155,7 +141,6 @@ class MockCursor(object):
|
||||
self.results = [(1, 0, 'host1', 5432, 'primary'), (2, 1, 'host2', 5432, 'primary')]
|
||||
else:
|
||||
self.results = [(None, None, None, None, None, None, None, None, None, None)]
|
||||
self.rowcount = len(self.results)
|
||||
|
||||
def fetchone(self):
|
||||
return self.results[0]
|
||||
@@ -174,20 +159,11 @@ class MockCursor(object):
|
||||
pass
|
||||
|
||||
|
||||
class MockConnectionInfo(object):
|
||||
|
||||
def parameter_status(self, param_name):
|
||||
if param_name == 'is_superuser':
|
||||
return 'on'
|
||||
return '0'
|
||||
|
||||
|
||||
class MockConnect(object):
|
||||
|
||||
server_version = 99999
|
||||
autocommit = False
|
||||
closed = 0
|
||||
info = MockConnectionInfo()
|
||||
|
||||
def cursor(self):
|
||||
return MockCursor(self)
|
||||
|
||||
+58
-185
@@ -3,6 +3,8 @@ import json
|
||||
import unittest
|
||||
import socket
|
||||
|
||||
import patroni.psycopg as psycopg
|
||||
|
||||
from http.server import HTTPServer
|
||||
from io import BytesIO as IO
|
||||
from mock import Mock, PropertyMock, patch
|
||||
@@ -11,43 +13,19 @@ from socketserver import ThreadingMixIn
|
||||
from patroni.api import RestApiHandler, RestApiServer
|
||||
from patroni.config import GlobalConfig
|
||||
from patroni.dcs import ClusterConfig, Member
|
||||
from patroni.exceptions import PostgresConnectionException
|
||||
from patroni.ha import _MemberStatus
|
||||
from patroni.manual_failover import ManualFailoverPrecheckStatus
|
||||
from patroni.psycopg import OperationalError
|
||||
from patroni.utils import ParseScheduleErrors, RetryFailedError, tzutc
|
||||
from patroni.utils import tzutc
|
||||
|
||||
from . import MockConnect, psycopg_connect
|
||||
from .test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader
|
||||
from . import psycopg_connect, MockCursor
|
||||
from .test_ha import get_cluster_initialized_without_leader
|
||||
|
||||
|
||||
future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
|
||||
postmaster_start_time = datetime.datetime.now(tzutc)
|
||||
|
||||
|
||||
class MockConnection:
|
||||
class MockPostgresql(object):
|
||||
|
||||
@staticmethod
|
||||
def get(*args):
|
||||
return psycopg_connect()
|
||||
|
||||
@staticmethod
|
||||
def query(sql, *params):
|
||||
return [(postmaster_start_time, 0, '', 0, '', False, postmaster_start_time, 'streaming', None,
|
||||
'[{"application_name":"walreceiver","client_addr":"1.2.3.4",'
|
||||
+ '"state":"streaming","sync_state":"async","sync_priority":0}]')]
|
||||
|
||||
|
||||
class MockConnectionPool:
|
||||
|
||||
@staticmethod
|
||||
def get(*args):
|
||||
return MockConnection()
|
||||
|
||||
|
||||
class MockPostgresql:
|
||||
|
||||
connection_pool = MockConnectionPool()
|
||||
name = 'test'
|
||||
state = 'running'
|
||||
role = 'primary'
|
||||
@@ -58,11 +36,14 @@ class MockPostgresql:
|
||||
pending_restart = True
|
||||
wal_name = 'wal'
|
||||
lsn_name = 'lsn'
|
||||
wal_flush = '_flush'
|
||||
POSTMASTER_START_TIME = 'pg_catalog.pg_postmaster_start_time()'
|
||||
TL_LSN = 'CASE WHEN pg_catalog.pg_is_in_recovery()'
|
||||
citus_handler = Mock()
|
||||
|
||||
@staticmethod
|
||||
def connection():
|
||||
return psycopg_connect()
|
||||
|
||||
@staticmethod
|
||||
def postmaster_start_time():
|
||||
return postmaster_start_time
|
||||
@@ -119,7 +100,7 @@ class MockHa(object):
|
||||
|
||||
@staticmethod
|
||||
def fetch_nodes_statuses(members):
|
||||
return [_MemberStatus(None, True, None, 0, {})]
|
||||
return [_MemberStatus(None, True, None, 0, 0, None, {}, False)]
|
||||
|
||||
@staticmethod
|
||||
def schedule_future_restart(data):
|
||||
@@ -141,9 +122,6 @@ class MockHa(object):
|
||||
def is_paused():
|
||||
return True
|
||||
|
||||
def has_members_eligible_to_promote(*args, **kwargs):
|
||||
return True
|
||||
|
||||
|
||||
class MockLogger(object):
|
||||
|
||||
@@ -509,7 +487,9 @@ class TestRestApiHandler(unittest.TestCase):
|
||||
|
||||
@patch('time.sleep', Mock())
|
||||
def test_RestApiServer_query(self):
|
||||
with patch.object(MockConnection, 'query', Mock(side_effect=RetryFailedError('bla'))):
|
||||
with patch.object(MockCursor, 'execute', Mock(side_effect=psycopg.OperationalError)):
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
|
||||
with patch.object(MockPostgresql, 'connection', Mock(side_effect=psycopg.OperationalError)):
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
|
||||
|
||||
@patch('time.sleep', Mock())
|
||||
@@ -520,185 +500,86 @@ class TestRestApiHandler(unittest.TestCase):
|
||||
|
||||
post = 'POST /switchover HTTP/1.0' + self._authorization + '\nContent-Length: '
|
||||
|
||||
# Invalid content
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}')
|
||||
response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SWITCHOVER_NO_LEADER.value[::-1])
|
||||
MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}')
|
||||
|
||||
# Empty content
|
||||
request = post + '0\n\n'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
# [Switchover without a candidate]
|
||||
|
||||
cluster.leader.name = 'postgresql1'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
request = post + '25\n\n{"leader": "postgresql1"}'
|
||||
|
||||
# No candidate in pause mode
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock, \
|
||||
patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
|
||||
with patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SWITCHOVER_PAUSE_NO_CANDIDATE.value[::-1])
|
||||
|
||||
# No healthy nodes to promote in both sync and async mode
|
||||
for is_synchronous_mode, response in (
|
||||
(True, ManualFailoverPrecheckStatus.NO_SYNC_CANDIDATE.value[0].format(action='switchover')),
|
||||
(False, ManualFailoverPrecheckStatus.ONLY_LEADER.value[0].format(action='switchover'))):
|
||||
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \
|
||||
patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
for is_synchronous_mode in (True, False):
|
||||
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)):
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(412, response)
|
||||
|
||||
# [Switchover to the candidate specified]
|
||||
cluster.leader.name = 'postgresql2'
|
||||
request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
# Candidate to promote is the same as the leader specified
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
request = post + '53\n\n{"leader": "postgresql2", "candidate": "postgresql2"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SWITCHOVER_TO_LEADER.value[::-1])
|
||||
|
||||
# Current leader is different from the one specified
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
cluster.leader.name = 'postgresql2'
|
||||
request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(
|
||||
ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER.value[1],
|
||||
ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER.value[0].format(leader='postgresql1',
|
||||
cluster_name='dummy'))
|
||||
|
||||
# Candidate to promote is not a sync standby/a member of the cluster
|
||||
cluster.leader.name = 'postgresql1'
|
||||
cluster.sync.matches.return_value = False
|
||||
for is_synchronous_mode, response in (
|
||||
(True, ManualFailoverPrecheckStatus.CANDIDATE_NOT_SYNC_STANDBY.value[0]),
|
||||
(False, ManualFailoverPrecheckStatus.CANDIDATE_NOT_MEMEBER.value[0].format(candidate="postgresql2",
|
||||
cluster_name='dummy'))):
|
||||
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \
|
||||
patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
for is_synchronous_mode in (True, False):
|
||||
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)):
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(412, response)
|
||||
|
||||
cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
|
||||
Member(0, 'postgresql2', 30, {'api_url': 'http'})]
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
# Cluster has no leader
|
||||
cluster.leader.name = None
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
request = post + '53\n\n{"leader": "postgresql1"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(
|
||||
ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER.value[1],
|
||||
ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER.value[0].format(leader='leader', cluster_name='dummy'))
|
||||
cluster.failover = None
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
cluster.leader.name = 'postgresql1'
|
||||
dcs.get_cluster.side_effect = [cluster]
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
# Failover key is empty in DCS
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
cluster.failover = None
|
||||
request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(503, 'Switchover failed')
|
||||
cluster2 = cluster.copy()
|
||||
cluster2.leader.name = 'postgresql0'
|
||||
cluster2.is_unlocked.return_value = False
|
||||
dcs.get_cluster.side_effect = [cluster, cluster2]
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
# Result polling failed
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
dcs.get_cluster.side_effect = [cluster]
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(503, 'Switchover status unknown')
|
||||
|
||||
# Switchover to a node different from the candidate specified
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
cluster2 = cluster.copy()
|
||||
cluster2.leader.name = 'postgresql0'
|
||||
cluster2.is_unlocked.return_value = False
|
||||
dcs.get_cluster.side_effect = [cluster, cluster2]
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(200, 'Switched over to "postgresql0" instead of "postgresql2"')
|
||||
|
||||
# Successful switchover to the candidate
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
cluster2.leader.name = 'postgresql2'
|
||||
dcs.get_cluster.side_effect = [cluster, cluster2]
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(200, 'Successfully switched over to "postgresql2"')
|
||||
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
dcs.manual_failover.return_value = False
|
||||
dcs.get_cluster.side_effect = None
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(503, 'failed to write failover key into DCS')
|
||||
cluster2.leader.name = 'postgresql2'
|
||||
dcs.get_cluster.side_effect = [cluster, cluster2]
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
dcs.get_cluster.side_effect = None
|
||||
dcs.manual_failover.return_value = False
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
dcs.manual_failover.return_value = True
|
||||
|
||||
# Candidate is not healthy to be promoted
|
||||
with patch.object(MockHa, 'has_members_eligible_to_promote', Mock(return_value=False)), \
|
||||
patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
with patch.object(MockHa, 'fetch_nodes_statuses', Mock(return_value=[])):
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(
|
||||
ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[1],
|
||||
ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[0].format(action='switchover'))
|
||||
|
||||
# [Scheduled switchover]
|
||||
|
||||
# Valid future date
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2",' + \
|
||||
' "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}'
|
||||
request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2",' +\
|
||||
' "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
with patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)), \
|
||||
patch.object(MockPatroni, 'dcs') as d:
|
||||
d.manual_failover.return_value = False
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(202, 'Switchover scheduled')
|
||||
|
||||
# Scheduled in pause mode
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock, \
|
||||
patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
|
||||
dcs.manual_failover.return_value = False
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SCHEDULED_SWITCHOVER_PAUSE.value[::-1])
|
||||
|
||||
# No timezone specified
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' + \
|
||||
' "scheduled_at": "6016-02-15T18:13:30.568224"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(
|
||||
ParseScheduleErrors.NO_TIMEZONE.value[1],
|
||||
ParseScheduleErrors.NO_TIMEZONE.value[0].format(action='switchover'))
|
||||
# Exception: No timezone specified
|
||||
request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' +\
|
||||
' "scheduled_at": "6016-02-15T18:13:30.568224"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
# Exception: Scheduled in the past
|
||||
request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "'
|
||||
|
||||
# Scheduled in the past
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
MockRestApiServer(RestApiHandler, request + '1016-02-15T18:13:30.568224+01:00"}')
|
||||
response_mock.assert_called_with(
|
||||
ParseScheduleErrors.SCHEDULED_IN_PAST.value[1],
|
||||
ParseScheduleErrors.SCHEDULED_IN_PAST.value[0].format(action='switchover'))
|
||||
MockRestApiServer(RestApiHandler, request + '1016-02-15T18:13:30.568224+01:00"}')
|
||||
|
||||
# Invalid date
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}')
|
||||
response_mock.assert_called_with(*ParseScheduleErrors.PARSING_ERROR.value[::-1])
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}'))
|
||||
|
||||
@patch.object(MockPatroni, 'dcs')
|
||||
def test_do_POST_failover(self, mock_dcs):
|
||||
def test_do_POST_failover(self):
|
||||
post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
|
||||
cluster = mock_dcs.get_cluster.return_value
|
||||
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
MockRestApiServer(RestApiHandler, post + '19\n\n{"leader":"leader"}')
|
||||
response_mock.assert_called_once_with(*ManualFailoverPrecheckStatus.FAILOVER_NO_CANDIDATE.value[::-1])
|
||||
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}')
|
||||
response_mock.assert_called_once_with(*ManualFailoverPrecheckStatus.SCHEDULED_FAILOVER.value[::-1])
|
||||
|
||||
# Candidate is not healthy to be promoted
|
||||
cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
|
||||
Member(0, 'postgresql2', 30, {'api_url': 'http'})]
|
||||
with patch.object(MockHa, 'has_members_eligible_to_promote', Mock(return_value=False)), \
|
||||
patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
MockRestApiServer(RestApiHandler, post + '27\n\n{"candidate":"postgresql2"}')
|
||||
response_mock.assert_called_with(
|
||||
ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[1],
|
||||
ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[0].format(action='failover'))
|
||||
MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}')
|
||||
MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}')
|
||||
|
||||
@patch.object(MockHa, 'is_leader', Mock(return_value=True))
|
||||
def test_do_POST_citus(self):
|
||||
@@ -780,11 +661,3 @@ class TestRestApiServer(unittest.TestCase):
|
||||
|
||||
def test_get_certificate_serial_number(self):
|
||||
self.assertIsNone(self.srv.get_certificate_serial_number())
|
||||
|
||||
def test_query(self):
|
||||
with patch.object(MockConnection, 'get', Mock(side_effect=OperationalError)):
|
||||
self.assertRaises(PostgresConnectionException, self.srv.query, 'SELECT 1')
|
||||
with patch.object(MockConnection, 'get', Mock(side_effect=[MockConnect(), OperationalError])), \
|
||||
patch.object(MockConnection, 'query') as mock_query:
|
||||
self.srv.query('SELECT 1')
|
||||
mock_query.assert_called_once_with('SELECT 1')
|
||||
|
||||
@@ -238,7 +238,8 @@ class TestBootstrap(BaseTestPostgresql):
|
||||
self.p.reload_config({'authentication': {'superuser': {'username': 'p', 'password': 'p'},
|
||||
'replication': {'username': 'r', 'password': 'r'},
|
||||
'rewind': {'username': 'rw', 'password': 'rw'}},
|
||||
'listen': '*', 'retry_timeout': 10, 'parameters': {'wal_level': '', 'hba_file': 'foo'}})
|
||||
'listen': '*', 'retry_timeout': 10,
|
||||
'parameters': {'wal_level': '', 'hba_file': 'foo', 'max_prepared_transactions': 10}})
|
||||
with patch.object(Postgresql, 'major_version', PropertyMock(return_value=110000)), \
|
||||
patch.object(Postgresql, 'restart', Mock()) as mock_restart:
|
||||
self.b.post_bootstrap({}, task)
|
||||
@@ -250,7 +251,7 @@ class TestBootstrap(BaseTestPostgresql):
|
||||
self.assertFalse(self.b.call_post_bootstrap({'post_init': '/bin/false'}))
|
||||
|
||||
mock_cancellable_subprocess_call.return_value = 0
|
||||
self.p.connection_pool._conn_kwargs.pop('user')
|
||||
self.p.config.superuser.pop('username')
|
||||
self.assertTrue(self.b.call_post_bootstrap({'post_init': '/bin/false'}))
|
||||
mock_cancellable_subprocess_call.assert_called()
|
||||
args, kwargs = mock_cancellable_subprocess_call.call_args
|
||||
@@ -258,7 +259,7 @@ class TestBootstrap(BaseTestPostgresql):
|
||||
self.assertEqual(args[0], ['/bin/false', 'dbname=postgres host=127.0.0.2 port=5432'])
|
||||
|
||||
mock_cancellable_subprocess_call.reset_mock()
|
||||
self.p.connection_pool._conn_kwargs.pop('host')
|
||||
self.p.config._local_address.pop('host')
|
||||
self.assertTrue(self.b.call_post_bootstrap({'post_init': '/bin/false'}))
|
||||
mock_cancellable_subprocess_call.assert_called()
|
||||
self.assertEqual(mock_cancellable_subprocess_call.call_args[0][0], ['/bin/false', 'dbname=postgres port=5432'])
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ class TestCitus(BaseTestPostgresql):
|
||||
def setUp(self):
|
||||
super(TestCitus, self).setUp()
|
||||
self.c = self.p.citus_handler
|
||||
self.p.connection_pool.conn_kwargs = {'host': 'localhost', 'dbname': 'postgres'}
|
||||
self.c.set_conn_kwargs({'host': 'localhost', 'dbname': 'postgres'})
|
||||
self.cluster = get_cluster_initialized_with_leader()
|
||||
self.cluster.workers[1] = self.cluster
|
||||
|
||||
|
||||
+24
-1
@@ -3,6 +3,7 @@ import sys
|
||||
import unittest
|
||||
import io
|
||||
|
||||
from copy import deepcopy
|
||||
from mock import MagicMock, Mock, patch
|
||||
from patroni.config import Config, ConfigParseError
|
||||
|
||||
@@ -22,7 +23,7 @@ class TestConfig(unittest.TestCase):
|
||||
self.assertFalse(self.config.set_dynamic_configuration({'foo': 'bar'}))
|
||||
self.assertTrue(self.config.set_dynamic_configuration({'standby_cluster': {}, 'postgresql': {
|
||||
'parameters': {'cluster_name': 1, 'hot_standby': 1, 'wal_keep_size': 1,
|
||||
'track_commit_timestamp': 1, 'wal_level': 1}}}))
|
||||
'track_commit_timestamp': 1, 'wal_level': 1, 'max_connections': '100'}}}))
|
||||
|
||||
def test_reload_local_configuration(self):
|
||||
os.environ.update({
|
||||
@@ -149,3 +150,25 @@ class TestConfig(unittest.TestCase):
|
||||
@patch('os.path.isdir', Mock(return_value=False))
|
||||
def test_invalid_path(self):
|
||||
self.assertRaises(ConfigParseError, Config, 'postgres0')
|
||||
|
||||
def test__process_postgresql_parameters(self):
|
||||
expected_params = {
|
||||
'f.oo': 'bar', # not in ConfigHandler.CMDLINE_OPTIONS
|
||||
'max_connections': 100, # IntValidator
|
||||
'wal_level': 'hot_standby', # EnumValidator
|
||||
}
|
||||
input_params = deepcopy(expected_params)
|
||||
|
||||
input_params['max_connections'] = '100'
|
||||
self.assertEqual(self.config._process_postgresql_parameters(input_params), expected_params)
|
||||
|
||||
expected_params['f.oo'] = input_params['f.oo'] = '100'
|
||||
self.assertEqual(self.config._process_postgresql_parameters(input_params), expected_params)
|
||||
|
||||
input_params['wal_level'] = 'cold_standby'
|
||||
expected_params.pop('wal_level')
|
||||
self.assertEqual(self.config._process_postgresql_parameters(input_params), expected_params)
|
||||
|
||||
input_params['max_connections'] = 10
|
||||
expected_params.pop('max_connections')
|
||||
self.assertEqual(self.config._process_postgresql_parameters(input_params), expected_params)
|
||||
|
||||
@@ -1,334 +0,0 @@
|
||||
import os
|
||||
import psutil
|
||||
import socket
|
||||
import unittest
|
||||
|
||||
from . import MockConnect, MockCursor, MockConnectionInfo
|
||||
from copy import deepcopy
|
||||
from mock import MagicMock, Mock, PropertyMock, mock_open, patch
|
||||
|
||||
from patroni.__main__ import main as _main
|
||||
from patroni.config import Config
|
||||
from patroni.config_generator import AbstractConfigGenerator, get_address
|
||||
|
||||
from patroni.utils import patch_config
|
||||
|
||||
from . import psycopg_connect
|
||||
|
||||
|
||||
@patch('patroni.psycopg.connect', psycopg_connect)
|
||||
@patch('socket.getaddrinfo', Mock(return_value=[(0, 0, 0, 0, ('1.9.8.4', 1984))]))
|
||||
@patch('builtins.open', MagicMock())
|
||||
@patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 16.2"))
|
||||
@patch('psutil.Process.exe', Mock(return_value='/bin/dir/from/running/postgres'))
|
||||
@patch('psutil.Process.__init__', Mock(return_value=None))
|
||||
class TestGenerateConfig(unittest.TestCase):
|
||||
|
||||
no_value_msg = '#FIXME'
|
||||
_HOSTNAME = socket.gethostname()
|
||||
_IP = sorted(socket.getaddrinfo(_HOSTNAME, 0, socket.AF_UNSPEC, socket.SOCK_STREAM, 0), key=lambda x: x[0])[0][4][0]
|
||||
|
||||
def setUp(self):
|
||||
self.maxDiff = None
|
||||
|
||||
os.environ['PATRONI_SCOPE'] = 'scope_from_env'
|
||||
os.environ['PATRONI_POSTGRESQL_BIN_DIR'] = '/bin/from/env'
|
||||
os.environ['PATRONI_SUPERUSER_USERNAME'] = 'su_user_from_env'
|
||||
os.environ['PATRONI_SUPERUSER_PASSWORD'] = 'su_pwd_from_env'
|
||||
os.environ['PATRONI_REPLICATION_USERNAME'] = 'repl_user_from_env'
|
||||
os.environ['PATRONI_REPLICATION_PASSWORD'] = 'repl_pwd_from_env'
|
||||
os.environ['PATRONI_REWIND_USERNAME'] = 'rewind_user_from_env'
|
||||
os.environ['PGUSER'] = 'pguser_from_env'
|
||||
os.environ['PGPASSWORD'] = 'pguser_pwd_from_env'
|
||||
os.environ['PATRONI_RESTAPI_CONNECT_ADDRESS'] = 'localhost:8080'
|
||||
os.environ['PATRONI_RESTAPI_LISTEN'] = 'localhost:8080'
|
||||
os.environ['PATRONI_POSTGRESQL_BIN_POSTGRES'] = 'custom_postgres_bin_from_env'
|
||||
|
||||
self.environ = deepcopy(os.environ)
|
||||
|
||||
dynamic_config = Config.get_default_config()
|
||||
dynamic_config['postgresql']['parameters'] = dict(dynamic_config['postgresql']['parameters'])
|
||||
del dynamic_config['standby_cluster']
|
||||
dynamic_config['postgresql']['parameters']['wal_keep_segments'] = 8
|
||||
dynamic_config['postgresql']['use_pg_rewind'] = True
|
||||
|
||||
self.config = {
|
||||
'scope': self.environ['PATRONI_SCOPE'],
|
||||
'name': self._HOSTNAME,
|
||||
'bootstrap': {
|
||||
'dcs': dynamic_config
|
||||
},
|
||||
'postgresql': {
|
||||
'connect_address': self.no_value_msg + ':5432',
|
||||
'data_dir': self.no_value_msg,
|
||||
'listen': self.no_value_msg + ':5432',
|
||||
'pg_hba': ['host all all all md5',
|
||||
f'host replication {self.environ["PATRONI_REPLICATION_USERNAME"]} all md5'],
|
||||
'authentication': {'superuser': {'username': self.environ['PATRONI_SUPERUSER_USERNAME'],
|
||||
'password': self.environ['PATRONI_SUPERUSER_PASSWORD']},
|
||||
'replication': {'username': self.environ['PATRONI_REPLICATION_USERNAME'],
|
||||
'password': self.environ['PATRONI_REPLICATION_PASSWORD']},
|
||||
'rewind': {'username': self.environ['PATRONI_REWIND_USERNAME']}},
|
||||
'bin_dir': self.environ['PATRONI_POSTGRESQL_BIN_DIR'],
|
||||
'bin_name': {'postgres': self.environ['PATRONI_POSTGRESQL_BIN_POSTGRES']},
|
||||
'parameters': {'password_encryption': 'md5'}
|
||||
},
|
||||
'restapi': {
|
||||
'connect_address': self.environ['PATRONI_RESTAPI_CONNECT_ADDRESS'],
|
||||
'listen': self.environ['PATRONI_RESTAPI_LISTEN']
|
||||
}
|
||||
}
|
||||
|
||||
def _set_running_instance_config_vals(self):
|
||||
# values are taken from tests/__init__.py
|
||||
conf = {
|
||||
'scope': 'my_cluster',
|
||||
'bootstrap': {
|
||||
'dcs': {
|
||||
'postgresql': {
|
||||
'parameters': {
|
||||
'max_connections': 42,
|
||||
'max_locks_per_transaction': 73,
|
||||
'max_replication_slots': 21,
|
||||
'max_wal_senders': 37,
|
||||
'wal_level': 'replica',
|
||||
'wal_keep_segments': None
|
||||
},
|
||||
'use_pg_rewind': None
|
||||
}
|
||||
}
|
||||
},
|
||||
'postgresql': {
|
||||
'connect_address': f'{self._IP}:bar',
|
||||
'listen': '6.6.6.6:1984',
|
||||
'data_dir': 'data',
|
||||
'bin_dir': '/bin/dir/from/running',
|
||||
'parameters': {
|
||||
'archive_command': 'my archive command',
|
||||
'hba_file': os.path.join('data', 'pg_hba.conf'),
|
||||
'ident_file': os.path.join('data', 'pg_ident.conf'),
|
||||
'password_encryption': None
|
||||
},
|
||||
'authentication': {
|
||||
'superuser': {
|
||||
'username': 'foobar',
|
||||
'password': 'qwerty',
|
||||
'channel_binding': 'prefer',
|
||||
'gssencmode': 'prefer',
|
||||
'sslmode': 'prefer'
|
||||
},
|
||||
'replication': {
|
||||
'username': self.no_value_msg,
|
||||
'password': self.no_value_msg
|
||||
},
|
||||
'rewind': None
|
||||
},
|
||||
}
|
||||
}
|
||||
patch_config(self.config, conf)
|
||||
|
||||
def _get_running_instance_open_res(self):
|
||||
hba_content = '\n'.join(self.config['postgresql']['pg_hba'] + ['#host all all all md5',
|
||||
' host all all all md5',
|
||||
'',
|
||||
'hostall all all md5'])
|
||||
ident_content = '\n'.join(['# something very interesting', ' '])
|
||||
|
||||
self.config['postgresql']['pg_hba'] += ['host all all all md5']
|
||||
return [
|
||||
mock_open(read_data=hba_content)(),
|
||||
mock_open(read_data=ident_content)(),
|
||||
mock_open(read_data='1984')(),
|
||||
mock_open()()
|
||||
]
|
||||
|
||||
@patch('os.makedirs')
|
||||
@patch('yaml.safe_dump')
|
||||
def test_generate_sample_config_pre_13_dir_creation(self, mock_config_dump, mock_makedir):
|
||||
with patch('sys.argv', ['patroni.py', '--generate-sample-config', '/foo/bar.yml']), \
|
||||
patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 9.4.3")) as pg_bin_mock, \
|
||||
self.assertRaises(SystemExit) as e:
|
||||
_main()
|
||||
self.assertEqual(e.exception.code, 0)
|
||||
self.assertEqual(self.config, mock_config_dump.call_args[0][0])
|
||||
mock_makedir.assert_called_once()
|
||||
pg_bin_mock.assert_called_once_with([os.path.join(self.environ['PATRONI_POSTGRESQL_BIN_DIR'],
|
||||
self.environ['PATRONI_POSTGRESQL_BIN_POSTGRES']),
|
||||
'--version'])
|
||||
|
||||
@patch('os.makedirs', Mock())
|
||||
@patch('yaml.safe_dump')
|
||||
def test_generate_sample_config_16(self, mock_config_dump):
|
||||
conf = {
|
||||
'bootstrap': {
|
||||
'dcs': {
|
||||
'postgresql': {
|
||||
'parameters': {
|
||||
'wal_keep_size': '128MB',
|
||||
'wal_keep_segments': None
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
'postgresql': {
|
||||
'parameters': {
|
||||
'password_encryption': 'scram-sha-256'
|
||||
},
|
||||
'pg_hba': ['host all all all scram-sha-256',
|
||||
f'host replication {self.environ["PATRONI_REPLICATION_USERNAME"]} all scram-sha-256'],
|
||||
'authentication': {
|
||||
'rewind': {
|
||||
'username': self.environ['PATRONI_REWIND_USERNAME'],
|
||||
'password': self.no_value_msg}
|
||||
},
|
||||
}
|
||||
}
|
||||
patch_config(self.config, conf)
|
||||
|
||||
with patch('sys.argv', ['patroni.py', '--generate-sample-config', '/foo/bar.yml']), \
|
||||
self.assertRaises(SystemExit) as e:
|
||||
_main()
|
||||
self.assertEqual(e.exception.code, 0)
|
||||
self.assertEqual(self.config, mock_config_dump.call_args[0][0])
|
||||
|
||||
@patch('os.makedirs', Mock())
|
||||
@patch('yaml.safe_dump')
|
||||
def test_generate_config_running_instance_16(self, mock_config_dump):
|
||||
self._set_running_instance_config_vals()
|
||||
|
||||
with patch('builtins.open', Mock(side_effect=self._get_running_instance_open_res())), \
|
||||
patch('sys.argv', ['patroni.py', '--generate-config',
|
||||
'--dsn', 'host=foo port=bar user=foobar password=qwerty']), \
|
||||
self.assertRaises(SystemExit) as e:
|
||||
_main()
|
||||
self.assertEqual(e.exception.code, 0)
|
||||
self.assertEqual(self.config, mock_config_dump.call_args[0][0])
|
||||
|
||||
@patch('os.makedirs', Mock())
|
||||
@patch('yaml.safe_dump')
|
||||
def test_generate_config_running_instance_16_connect_from_env(self, mock_config_dump):
|
||||
self._set_running_instance_config_vals()
|
||||
# su auth params and connect host from env
|
||||
os.environ['PGCHANNELBINDING'] = \
|
||||
self.config['postgresql']['authentication']['superuser']['channel_binding'] = 'disable'
|
||||
|
||||
conf = {
|
||||
'scope': 'my_cluster',
|
||||
'bootstrap': {
|
||||
'dcs': {
|
||||
'postgresql': {
|
||||
'parameters': {
|
||||
'max_connections': 42,
|
||||
'max_locks_per_transaction': 73,
|
||||
'max_replication_slots': 21,
|
||||
'max_wal_senders': 37,
|
||||
'wal_level': 'replica',
|
||||
'wal_keep_segments': None
|
||||
},
|
||||
'use_pg_rewind': None
|
||||
}
|
||||
}
|
||||
},
|
||||
'postgresql': {
|
||||
'connect_address': f'{self._IP}:1984',
|
||||
'authentication': {
|
||||
'superuser': {
|
||||
'username': self.environ['PGUSER'],
|
||||
'password': self.environ['PGPASSWORD'],
|
||||
'gssencmode': None,
|
||||
'sslmode': None
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
patch_config(self.config, conf)
|
||||
|
||||
with patch('builtins.open', Mock(side_effect=self._get_running_instance_open_res())), \
|
||||
patch('sys.argv', ['patroni.py', '--generate-config']), \
|
||||
patch.object(MockConnect, 'server_version', PropertyMock(return_value=160000)), \
|
||||
self.assertRaises(SystemExit) as e:
|
||||
_main()
|
||||
self.assertEqual(e.exception.code, 0)
|
||||
self.assertEqual(self.config, mock_config_dump.call_args[0][0])
|
||||
|
||||
def test_generate_config_running_instance_errors(self):
|
||||
# 1. Wrong DSN format
|
||||
with patch('sys.argv', ['patroni.py', '--generate-config', '--dsn', 'host:foo port:bar user:foobar']), \
|
||||
self.assertRaises(SystemExit) as e:
|
||||
_main()
|
||||
self.assertIn('Failed to parse DSN string', e.exception.code)
|
||||
|
||||
# 2. User is not a superuser
|
||||
with patch('sys.argv', ['patroni.py',
|
||||
'--generate-config', '--dsn', 'host=foo port=bar user=foobar password=pwd_from_dsn']), \
|
||||
patch.object(MockCursor, 'rowcount', PropertyMock(return_value=0), create=True), \
|
||||
patch.object(MockConnectionInfo, 'parameter_status', Mock(return_value='off')), \
|
||||
self.assertRaises(SystemExit) as e:
|
||||
_main()
|
||||
self.assertIn('The provided user does not have superuser privilege', e.exception.code)
|
||||
|
||||
# 3. Error while calling postgres --version
|
||||
with patch('subprocess.check_output', Mock(side_effect=OSError)), \
|
||||
patch('sys.argv', ['patroni.py', '--generate-sample-config']), \
|
||||
self.assertRaises(SystemExit) as e:
|
||||
_main()
|
||||
self.assertIn('Failed to get postgres version:', e.exception.code)
|
||||
|
||||
with patch('sys.argv', ['patroni.py', '--generate-config']):
|
||||
|
||||
# 4. empty postmaster.pid
|
||||
with patch('builtins.open', Mock(side_effect=[mock_open(read_data='hba_content')(),
|
||||
mock_open(read_data='ident_content')(),
|
||||
mock_open(read_data='')()])), \
|
||||
self.assertRaises(SystemExit) as e:
|
||||
_main()
|
||||
self.assertIn('Failed to obtain postmaster pid from postmaster.pid file', e.exception.code)
|
||||
|
||||
# 5. Failed to open postmaster.pid
|
||||
with patch('builtins.open', Mock(side_effect=[mock_open(read_data='hba_content')(),
|
||||
mock_open(read_data='ident_content')(),
|
||||
OSError])), \
|
||||
self.assertRaises(SystemExit) as e:
|
||||
_main()
|
||||
self.assertIn('Error while reading postmaster.pid file', e.exception.code)
|
||||
|
||||
# 6. Invalid postmaster pid
|
||||
with patch('builtins.open', Mock(side_effect=[mock_open(read_data='hba_content')(),
|
||||
mock_open(read_data='ident_content')(),
|
||||
mock_open(read_data='1984')()])), \
|
||||
patch('psutil.Process.__init__', Mock(return_value=None)), \
|
||||
patch('psutil.Process.exe', Mock(side_effect=psutil.NoSuchProcess(1984))), \
|
||||
self.assertRaises(SystemExit) as e:
|
||||
_main()
|
||||
self.assertIn("Obtained postmaster pid doesn't exist", e.exception.code)
|
||||
|
||||
# 7. Failed to open pg_hba
|
||||
with patch('builtins.open', Mock(side_effect=OSError)), \
|
||||
self.assertRaises(SystemExit) as e:
|
||||
_main()
|
||||
self.assertIn('Failed to read pg_hba.conf', e.exception.code)
|
||||
|
||||
# 8. Failed to open pg_ident
|
||||
with patch('builtins.open', Mock(side_effect=[mock_open(read_data='hba_content')(), OSError])), \
|
||||
self.assertRaises(SystemExit) as e:
|
||||
_main()
|
||||
self.assertIn('Failed to read pg_ident.conf', e.exception.code)
|
||||
|
||||
# 9. Failed PG connecttion
|
||||
from . import psycopg
|
||||
with patch('patroni.psycopg.connect', side_effect=psycopg.Error), \
|
||||
self.assertRaises(SystemExit) as e:
|
||||
_main()
|
||||
self.assertIn('Failed to establish PostgreSQL connection', e.exception.code)
|
||||
|
||||
# 10. An unexpected error
|
||||
with patch.object(AbstractConfigGenerator, '__init__', side_effect=psycopg.Error), \
|
||||
self.assertRaises(SystemExit) as e:
|
||||
_main()
|
||||
self.assertIn('Unexpected exception', e.exception.code)
|
||||
|
||||
def test_get_address(self):
|
||||
with patch('socket.getaddrinfo', Mock(side_effect=Exception)), \
|
||||
patch('logging.warning') as mock_warning:
|
||||
self.assertEqual(get_address(), (self.no_value_msg, self.no_value_msg))
|
||||
self.assertIn('Failed to obtain address: %r', mock_warning.call_args_list[0][0])
|
||||
@@ -197,10 +197,9 @@ class TestConsul(unittest.TestCase):
|
||||
|
||||
@patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
|
||||
def test_delete_leader(self):
|
||||
leader = self.c.get_cluster().leader
|
||||
self.c.delete_leader(leader)
|
||||
self.c.delete_leader()
|
||||
self.c._name = 'other'
|
||||
self.c.delete_leader(leader)
|
||||
self.c.delete_leader()
|
||||
|
||||
@patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
|
||||
def test_initialize(self):
|
||||
|
||||
+142
-255
@@ -6,14 +6,12 @@ import unittest
|
||||
from click.testing import CliRunner
|
||||
from datetime import datetime, timedelta
|
||||
from mock import patch, Mock, PropertyMock
|
||||
from patroni.config import GlobalConfig
|
||||
from patroni.ctl import ctl, load_config, output_members, get_dcs, parse_dcs, \
|
||||
get_all_members, get_any_member, get_cursor, query_member, PatroniCtlException, apply_config_changes, \
|
||||
format_config_for_editing, show_diff, invoke_editor, format_pg_version, CONFIG_FILE_PATH, PatronictlPrettyTable
|
||||
from patroni.dcs.etcd import AbstractEtcdClientWithFailover, Cluster, Failover
|
||||
from patroni.manual_failover import ManualFailoverPrecheckStatus
|
||||
from patroni.psycopg import OperationalError
|
||||
from patroni.utils import ParseScheduleErrors, tzutc
|
||||
from patroni.utils import tzutc
|
||||
from prettytable import PrettyTable, ALL
|
||||
from urllib3 import PoolManager
|
||||
|
||||
@@ -23,24 +21,13 @@ from .test_ha import get_cluster_initialized_without_leader, get_cluster_initial
|
||||
get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader, get_cluster, Member
|
||||
|
||||
|
||||
DEFAULT_CONFIG = {
|
||||
'scope': 'alpha',
|
||||
'restapi': {'listen': '::', 'certfile': 'a'},
|
||||
'ctl': {'certfile': 'a'},
|
||||
'etcd': {'host': 'localhost:2379'},
|
||||
'citus': {'database': 'citus', 'group': 0},
|
||||
'postgresql': {'data_dir': '.', 'pgpass': './pgpass', 'parameters': {}, 'retry_timeout': 5}
|
||||
}
|
||||
|
||||
|
||||
@patch('patroni.ctl.load_config', Mock(return_value=DEFAULT_CONFIG))
|
||||
@patch('patroni.ctl.load_config', Mock(return_value={
|
||||
'scope': 'alpha', 'restapi': {'listen': '::', 'certfile': 'a'}, 'ctl': {'certfile': 'a'},
|
||||
'etcd': {'host': 'localhost:2379'}, 'citus': {'database': 'citus', 'group': 0},
|
||||
'postgresql': {'data_dir': '.', 'pgpass': './pgpass', 'parameters': {}, 'retry_timeout': 5}}))
|
||||
class TestCtl(unittest.TestCase):
|
||||
TEST_ROLES = ('master', 'primary', 'leader')
|
||||
|
||||
SCHEDULED_TS = '2055-01-01T12:00:00+01:00'
|
||||
SCHEDULED_TS_NO_TZ = '2055-01-01T12:00:00'
|
||||
SCHEDULED_TS_INVALID = '2055-02-30T12:00:00'
|
||||
|
||||
@patch('socket.getaddrinfo', socket_getaddrinfo)
|
||||
@patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379']))
|
||||
def setUp(self):
|
||||
@@ -81,6 +68,21 @@ class TestCtl(unittest.TestCase):
|
||||
|
||||
self.assertIsNotNone(get_cursor({}, get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, role='any'))
|
||||
|
||||
# Mutually exclusive options
|
||||
with self.assertRaises(PatroniCtlException) as e:
|
||||
get_cursor({}, get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, member_name='other',
|
||||
role='replica')
|
||||
|
||||
self.assertEqual(str(e.exception), '--role and --member are mutually exclusive options')
|
||||
|
||||
# Invalid member provided
|
||||
self.assertIsNone(get_cursor({}, get_cluster_initialized_with_leader(), None, {'dbname': 'foo'},
|
||||
member_name='invalid'))
|
||||
|
||||
# Valid member provided
|
||||
self.assertIsNotNone(get_cursor({}, get_cluster_initialized_with_leader(), None, {'dbname': 'foo'},
|
||||
member_name='other'))
|
||||
|
||||
def test_parse_dcs(self):
|
||||
assert parse_dcs(None) is None
|
||||
assert parse_dcs('localhost') == {'etcd': {'host': 'localhost:2379'}}
|
||||
@@ -109,194 +111,91 @@ class TestCtl(unittest.TestCase):
|
||||
mock_get_dcs.return_value = self.e
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
|
||||
mock_get_dcs.return_value.set_failover_value = Mock()
|
||||
|
||||
# Confirm
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
||||
self.assertEqual(result.exit_code, 0)
|
||||
assert 'leader' in result.output
|
||||
|
||||
# Abort
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
|
||||
input='leader\nother\n2300-01-01T12:23:00\ny')
|
||||
assert result.exit_code == 0
|
||||
|
||||
with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||
'--force', '--scheduled', '2015-01-01T12:00:00'])
|
||||
assert result.exit_code == 1
|
||||
|
||||
# Aborting switchover, as we answer NO to the confirmation
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\nN')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
assert result.exit_code == 1
|
||||
|
||||
# Aborting scheduled switchover, as we answer NO to the confirmation
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||
'--scheduled', '2015-01-01T12:00:00+01:00'], input='leader\nother\n\nN')
|
||||
assert result.exit_code == 1
|
||||
|
||||
# Target and source are equal
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nleader\n\ny')
|
||||
assert result.exit_code == 1
|
||||
|
||||
# Reality is not part of this cluster
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nReality\n\ny')
|
||||
assert result.exit_code == 1
|
||||
|
||||
# Without a candidate with --force option
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force'])
|
||||
self.assertEqual(result.exit_code, 0)
|
||||
assert 'Member' in result.output
|
||||
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||
'--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
|
||||
assert result.exit_code == 0
|
||||
|
||||
# Invalid timestamp
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force', '--scheduled', 'invalid'])
|
||||
assert result.exit_code != 0
|
||||
|
||||
# Invalid timestamp
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||
'--force', '--scheduled', '2115-02-30T12:00:00+01:00'])
|
||||
assert result.exit_code != 0
|
||||
|
||||
# Specifying wrong leader
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='dummy')
|
||||
assert result.exit_code == 1
|
||||
|
||||
with patch.object(PoolManager, 'request', Mock(side_effect=Exception)):
|
||||
# Non-responding patroni
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
|
||||
input='leader\nother\n2300-01-01T12:23:00\ny')
|
||||
assert 'falling back to DCS' in result.output
|
||||
|
||||
with patch.object(PoolManager, 'request') as mocked:
|
||||
mocked.return_value.status = 500
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
||||
assert 'Switchover failed' in result.output
|
||||
|
||||
mocked.return_value.status = 501
|
||||
mocked.return_value.data = b'Server does not support this operation'
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
||||
assert 'Switchover failed' in result.output
|
||||
|
||||
# No members available
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn('No candidates found to switchover to', result.output)
|
||||
assert result.exit_code == 1
|
||||
|
||||
# No leader available
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn('This cluster has no leader', result.output)
|
||||
|
||||
# Citus cluster, no group number specified
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force'], input='\n')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn('For Citus clusters the --group must me specified', result.output)
|
||||
|
||||
# [Scheduled]
|
||||
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
|
||||
|
||||
# Scheduled (confirm)
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
|
||||
input=f'leader\nother\n{self.SCHEDULED_TS}\ny')
|
||||
self.assertEqual(result.exit_code, 0)
|
||||
self.assertIn(f'Are you sure you want to schedule a switchover in the cluster dummy '
|
||||
f'at {self.SCHEDULED_TS}, demoting current leader', result.output)
|
||||
|
||||
# Scheduled (abort)
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||
'--scheduled', self.SCHEDULED_TS], input='leader\nother\n\nN')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
|
||||
# Scheduled with --force option
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||
'--force', '--scheduled', self.SCHEDULED_TS])
|
||||
self.assertEqual(result.exit_code, 0)
|
||||
|
||||
# Scheduled in pause mode
|
||||
with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||
'--force', '--scheduled', self.SCHEDULED_TS])
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn(ManualFailoverPrecheckStatus.SCHEDULED_SWITCHOVER_PAUSE.value[0], result.output)
|
||||
|
||||
# Invalid timestamp with force
|
||||
result = self.runner.invoke(ctl,['switchover', 'dummy', '--group', '0', '--force', '--scheduled',
|
||||
self.SCHEDULED_TS_INVALID])
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn('Unable to parse scheduled timestamp', result.output)
|
||||
|
||||
# Invalid timestamp
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||
'--force', '--scheduled', self.SCHEDULED_TS_INVALID])
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn('Unable to parse scheduled timestamp', result.output)
|
||||
|
||||
# Invalid timestamp - no timezone
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||
'--force', '--scheduled', self.SCHEDULED_TS_NO_TZ])
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn(ParseScheduleErrors.NO_TIMEZONE.value[0].format(action='switchover'), result.output)
|
||||
|
||||
# [Other erroneous combinations]
|
||||
|
||||
# No candidate in pause mode
|
||||
with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\n\n\ny')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn(ManualFailoverPrecheckStatus.SWITCHOVER_PAUSE_NO_CANDIDATE.value[0], result.output)
|
||||
|
||||
# Target and source are equal
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nleader\n\ny')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn(ManualFailoverPrecheckStatus.SWITCHOVER_TO_LEADER.value[0], result.output)
|
||||
|
||||
# Candidate is not a member of the cluster
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nReality\n\ny')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn(ManualFailoverPrecheckStatus.CANDIDATE_NOT_MEMEBER.value[0].format(candidate='Reality',
|
||||
cluster_name='dummy'),
|
||||
result.output)
|
||||
|
||||
# Specifying wrong leader
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='dummy')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn(
|
||||
ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER.value[0].format(leader='dummy',
|
||||
cluster_name='dummy'),
|
||||
result.output)
|
||||
|
||||
mock_get_dcs.return_value.get_cluster = Mock(
|
||||
return_value=get_cluster_initialized_with_leader(sync=('leader', 'other')))
|
||||
|
||||
# Candidate is not a sync standby
|
||||
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=True)):
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\notherMember\n\ny')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn(ManualFailoverPrecheckStatus.CANDIDATE_NOT_SYNC_STANDBY.value[0], result.output)
|
||||
|
||||
# No healthy nodes to promote in sync mode
|
||||
mock_get_dcs.return_value.get_cluster = Mock(return_value=get_cluster_initialized_with_leader(sync=('leader')))
|
||||
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=True)):
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force'])
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn(ManualFailoverPrecheckStatus.NO_SYNC_CANDIDATE.value[0].format(action='switchover'),
|
||||
result.output)
|
||||
|
||||
# No healthy nodes to promote in async mode
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
|
||||
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=False)):
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force'])
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn(ManualFailoverPrecheckStatus.ONLY_LEADER.value[0].format(action='switchover'),
|
||||
result.output)
|
||||
|
||||
# Cluster has no leader
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--leader', 'leader', '--force'])
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn(
|
||||
ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER.value[0].format(leader='leader', cluster_name='dummy'),
|
||||
result.output)
|
||||
|
||||
# [Errors while sending Patroni REST API request]
|
||||
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
|
||||
|
||||
with patch.object(PoolManager, 'request', Mock(side_effect=Exception)):
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
|
||||
input=f'leader\nother\n{self.SCHEDULED_TS}\ny')
|
||||
self.assertIn('falling back to DCS', result.output)
|
||||
|
||||
with patch.object(PoolManager, 'request') as mock_api_request:
|
||||
mock_api_request.return_value.status = 500
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
||||
self.assertIn('Switchover failed', result.output)
|
||||
|
||||
mock_api_request.return_value.status = 501
|
||||
mock_api_request.return_value.data = b'Server does not support this operation'
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
||||
self.assertIn('Switchover failed', result.output)
|
||||
assert result.exit_code == 1
|
||||
|
||||
@patch('patroni.ctl.get_dcs')
|
||||
@patch.object(PoolManager, 'request', Mock(return_value=MockResponse()))
|
||||
@patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
|
||||
def test_failover(self, mock_get_dcs):
|
||||
mock_get_dcs.return_value.set_failover_value = Mock()
|
||||
|
||||
# No candidate specified
|
||||
mock_get_dcs.return_value = self.e
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
|
||||
mock_get_dcs.return_value.set_failover_value = Mock()
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--force'], input='\n')
|
||||
assert 'For Citus clusters the --group must me specified' in result.output
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='0\n')
|
||||
self.assertIn(ManualFailoverPrecheckStatus.FAILOVER_NO_CANDIDATE.value[0], result.output)
|
||||
|
||||
# Failover to an async member in sync mode (confirm)
|
||||
cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
|
||||
|
||||
# Temp test to check a fallback to switchover if leader is specified
|
||||
with patch('patroni.ctl._do_failover_or_switchover') as failover_func_mock:
|
||||
result = self.runner.invoke(ctl, ['failover', '--leader', 'leader', 'dummy'], input='0\n')
|
||||
self.assertIn('Supplying a leader name using this command is deprecated', result.output)
|
||||
failover_func_mock.assert_called_once_with(
|
||||
DEFAULT_CONFIG, 'switchover', 'dummy', None, 'leader', None, False)
|
||||
|
||||
# Failover to an async member in sync mode (confirm)
|
||||
cluster.members.append(Member(0, 'async', 28, {'api_url': 'http://127.0.0.1:8012/patroni'}))
|
||||
cluster.config.data['synchronous_mode'] = True
|
||||
mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--group', '0', '--candidate', 'async'], input='y\ny')
|
||||
self.assertIn('Are you sure you want to failover to the asynchronous node async', result.output)
|
||||
|
||||
# Failover to an async member in sync mode (abort)
|
||||
mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--group', '0', '--candidate', 'async'], input='N')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
assert 'Failover could be performed only to a specific candidate' in result.output
|
||||
|
||||
@patch('patroni.dcs.dcs_modules', Mock(return_value=['patroni.dcs.dummy', 'patroni.dcs.etcd']))
|
||||
def test_get_dcs(self):
|
||||
@@ -347,11 +246,17 @@ class TestCtl(unittest.TestCase):
|
||||
rows = query_member({}, None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
|
||||
|
||||
with patch('patroni.ctl.get_cursor', Mock(return_value=None)):
|
||||
# No role nor member given -- generic message
|
||||
rows = query_member({}, None, None, None, None, None, 'SELECT pg_catalog.pg_is_in_recovery()', {})
|
||||
self.assertTrue('No connection to' in str(rows))
|
||||
self.assertTrue('No connection is available' in str(rows))
|
||||
|
||||
rows = query_member({}, None, None, None, 'foo', 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
|
||||
self.assertTrue('No connection to' in str(rows))
|
||||
# Member given -- message pointing to member
|
||||
rows = query_member({}, None, None, None, 'foo', None, 'SELECT pg_catalog.pg_is_in_recovery()', {})
|
||||
self.assertTrue('No connection to member foo' in str(rows))
|
||||
|
||||
# Role given -- message pointing to role
|
||||
rows = query_member({}, None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
|
||||
self.assertTrue('No connection to role replica' in str(rows))
|
||||
|
||||
with patch('patroni.ctl.get_cursor', Mock(side_effect=OperationalError('bla'))):
|
||||
rows = query_member({}, None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
|
||||
@@ -389,9 +294,12 @@ class TestCtl(unittest.TestCase):
|
||||
|
||||
@patch.object(PoolManager, 'request')
|
||||
@patch('patroni.ctl.get_dcs')
|
||||
def test_reinit(self, mock_get_dcs, mock_post):
|
||||
def test_restart_reinit(self, mock_get_dcs, mock_post):
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
|
||||
mock_post.return_value.status = 503
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha'], input='now\ny\n')
|
||||
assert 'Failed: restart for' in result.output
|
||||
assert result.exit_code == 0
|
||||
|
||||
result = self.runner.invoke(ctl, ['reinit', 'alpha'], input='y')
|
||||
assert result.exit_code == 1
|
||||
@@ -400,88 +308,67 @@ class TestCtl(unittest.TestCase):
|
||||
result = self.runner.invoke(ctl, ['reinit', 'alpha', 'other'], input='y\ny')
|
||||
assert result.exit_code == 0
|
||||
|
||||
@patch.object(PoolManager, 'request')
|
||||
@patch('patroni.ctl.get_dcs')
|
||||
def test_restart(self, mock_get_dcs, mock_post):
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
|
||||
mock_post.return_value.status = 200
|
||||
|
||||
# Successful restart
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha'], input='now\ny\n')
|
||||
self.assertEqual(result.exit_code, 0)
|
||||
|
||||
# Aborted
|
||||
# Aborted restart
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha'], input='now\nN')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
assert result.exit_code == 1
|
||||
|
||||
# With pending the flag
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force'])
|
||||
self.assertEqual(result.exit_code, 0)
|
||||
assert result.exit_code == 0
|
||||
|
||||
# Aborted scheduled restart
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha', '--scheduled', '2019-10-01T14:30'], input='N')
|
||||
assert result.exit_code == 1
|
||||
|
||||
# Not a member
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha', 'dummy', '--any'], input='now\ny')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn('Not a single cluster member among provided members', result.output)
|
||||
|
||||
# Not a member with the specified role
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--role', 'primary'], input='now\ny')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn('No primary among provided members', result.output)
|
||||
assert result.exit_code == 1
|
||||
|
||||
# Wrong pg version
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha', '--any', '--pg-version', '9.1'], input='now\ny')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn('Error: Invalid PostgreSQL version format', result.output)
|
||||
assert 'Error: Invalid PostgreSQL version format' in result.output
|
||||
assert result.exit_code == 1
|
||||
|
||||
# Restart with timeout
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force', '--timeout', '10min'])
|
||||
self.assertEqual(result.exit_code, 0)
|
||||
assert result.exit_code == 0
|
||||
|
||||
# Scheduled restart
|
||||
# normal restart, the schedule is actually parsed, but not validated in patronictl
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
|
||||
assert 'Failed: flush scheduled restart' in result.output
|
||||
|
||||
# Aborted scheduled restart
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha', '--scheduled', self.SCHEDULED_TS], input='N')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
|
||||
# Error parsing scheduled flag value (no tz)
|
||||
result = self.runner.invoke(ctl,
|
||||
['restart', 'alpha', 'other', '--force', '--scheduled', self.SCHEDULED_TS_NO_TZ])
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn(ParseScheduleErrors.NO_TIMEZONE.value[0].format(action='restart'), result.output)
|
||||
|
||||
# Error parsing scheduled flag value (invalid date)
|
||||
result = self.runner.invoke(ctl,
|
||||
['restart', 'alpha', 'other', '--force', '--scheduled', self.SCHEDULED_TS_INVALID])
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn('Unable to parse scheduled timestamp', result.output)
|
||||
|
||||
# Successfully scheduled restart
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha', '--scheduled', self.SCHEDULED_TS], input='Y')
|
||||
self.assertEqual(result.exit_code, 0)
|
||||
self.assertIn('Success: restart on member other', result.output)
|
||||
|
||||
# Not possible to schedule in pause mode
|
||||
with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
|
||||
result = self.runner.invoke(ctl,
|
||||
['restart', 'alpha', 'other', '--force', '--scheduled', self.SCHEDULED_TS])
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn("Can't schedule restart in the paused state", result.output)
|
||||
['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
|
||||
assert result.exit_code == 1
|
||||
|
||||
# Force restart with restart already scheduled
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force', '--scheduled', self.SCHEDULED_TS])
|
||||
self.assertEqual(result.exit_code, 0)
|
||||
# force restart with restart already present
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
|
||||
assert result.exit_code == 0
|
||||
|
||||
ctl_args = ['restart', 'alpha', '--pg-version', '99.0', '--scheduled', '2300-10-01T14:30']
|
||||
# normal restart, the schedule is actually parsed, but not validated in patronictl
|
||||
mock_post.return_value.status = 200
|
||||
result = self.runner.invoke(ctl, ctl_args, input='y')
|
||||
assert result.exit_code == 0
|
||||
|
||||
# get restart with the non-200 return code
|
||||
ctl_args = ['restart', 'alpha', '--pg-version', '99.0', '--scheduled', self.SCHEDULED_TS]
|
||||
for code, output in [
|
||||
(204, 'Failed: restart for member other, status code=204'),
|
||||
(202, 'Success: restart scheduled'),
|
||||
(409, 'Failed: another restart is already')
|
||||
]:
|
||||
mock_post.return_value.status = code
|
||||
result = self.runner.invoke(ctl, ctl_args, input='y')
|
||||
self.assertEqual(result.exit_code, 0)
|
||||
self.assertIn(output, result.output)
|
||||
# normal restart, the schedule is actually parsed, but not validated in patronictl
|
||||
mock_post.return_value.status = 204
|
||||
result = self.runner.invoke(ctl, ctl_args, input='y')
|
||||
assert result.exit_code == 0
|
||||
|
||||
# get restart with the non-200 return code
|
||||
# normal restart, the schedule is actually parsed, but not validated in patronictl
|
||||
mock_post.return_value.status = 202
|
||||
result = self.runner.invoke(ctl, ctl_args, input='y')
|
||||
assert 'Success: restart scheduled' in result.output
|
||||
assert result.exit_code == 0
|
||||
|
||||
# get restart with the non-200 return code
|
||||
# normal restart, the schedule is actually parsed, but not validated in patronictl
|
||||
mock_post.return_value.status = 409
|
||||
result = self.runner.invoke(ctl, ctl_args, input='y')
|
||||
assert 'Failed: another restart is already' in result.output
|
||||
assert result.exit_code == 0
|
||||
|
||||
@patch('patroni.ctl.get_dcs')
|
||||
def test_remove(self, mock_get_dcs):
|
||||
|
||||
+1
-1
@@ -313,7 +313,7 @@ class TestEtcd(unittest.TestCase):
|
||||
self.assertFalse(self.etcd.cancel_initialization())
|
||||
|
||||
def test_delete_leader(self):
|
||||
self.assertFalse(self.etcd.delete_leader(self.etcd.get_cluster().leader))
|
||||
self.assertFalse(self.etcd.delete_leader())
|
||||
|
||||
def test_delete_cluster(self):
|
||||
self.assertFalse(self.etcd.delete_cluster())
|
||||
|
||||
+3
-4
@@ -298,10 +298,9 @@ class TestEtcd3(BaseTestEtcd3):
|
||||
self.etcd3.cancel_initialization()
|
||||
|
||||
def test_delete_leader(self):
|
||||
leader = self.etcd3.get_cluster().leader
|
||||
self.etcd3.delete_leader(leader)
|
||||
self.etcd3.delete_leader()
|
||||
self.etcd3._name = 'other'
|
||||
self.etcd3.delete_leader(leader)
|
||||
self.etcd3.delete_leader()
|
||||
|
||||
def test_delete_cluster(self):
|
||||
self.etcd3.delete_cluster()
|
||||
@@ -313,7 +312,7 @@ class TestEtcd3(BaseTestEtcd3):
|
||||
self.etcd3.set_sync_state_value('', 1)
|
||||
|
||||
def test_delete_sync_state(self):
|
||||
self.etcd3.delete_sync_state('1')
|
||||
self.etcd3.delete_sync_state()
|
||||
|
||||
def test_watch(self):
|
||||
self.etcd3.set_ttl(10)
|
||||
|
||||
+140
-315
@@ -99,9 +99,7 @@ def get_node_status(reachable=True, in_recovery=True, dcs_last_seen=0,
|
||||
tags = {}
|
||||
if nofailover:
|
||||
tags['nofailover'] = True
|
||||
return _MemberStatus(e, reachable, in_recovery, wal_position,
|
||||
{'tags': tags, 'watchdog_failed': watchdog_failed,
|
||||
'dcs_last_seen': dcs_last_seen, 'timeline': timeline})
|
||||
return _MemberStatus(e, reachable, in_recovery, dcs_last_seen, timeline, wal_position, tags, watchdog_failed)
|
||||
return fetch_node_status
|
||||
|
||||
|
||||
@@ -164,7 +162,7 @@ def run_async(self, func, args=()):
|
||||
|
||||
|
||||
@patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
|
||||
@patch.object(Postgresql, 'is_primary', Mock(return_value=True))
|
||||
@patch.object(Postgresql, 'is_leader', Mock(return_value=True))
|
||||
@patch.object(Postgresql, 'timeline_wal_position', Mock(return_value=(1, 10, 1)))
|
||||
@patch.object(Postgresql, '_cluster_info_state_get', Mock(return_value=10))
|
||||
@patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
|
||||
@@ -226,7 +224,7 @@ class TestHa(PostgresInit):
|
||||
@patch.object(Postgresql, 'received_timeline', Mock(return_value=None))
|
||||
def test_touch_member(self):
|
||||
self.p._major_version = 110000
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.p.timeline_wal_position = Mock(return_value=(0, 1, 0))
|
||||
self.p.replica_cached_timeline = Mock(side_effect=Exception)
|
||||
with patch.object(Postgresql, '_cluster_info_state_get', Mock(return_value='streaming')):
|
||||
@@ -322,7 +320,7 @@ class TestHa(PostgresInit):
|
||||
@patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
|
||||
@patch.object(Rewind, 'can_rewind', PropertyMock(return_value=True))
|
||||
def test_crash_recovery_before_rewind(self):
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.p.is_running = false
|
||||
self.p.controldata = lambda: {'Database cluster state': 'in archive recovery',
|
||||
'Database system identifier': SYSID}
|
||||
@@ -367,7 +365,7 @@ class TestHa(PostgresInit):
|
||||
|
||||
@patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
|
||||
def test_start_as_readonly(self):
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.p.is_healthy = true
|
||||
self.ha.has_lock = true
|
||||
self.p.controldata = lambda: {'Database cluster state': 'in production', 'Database system identifier': SYSID}
|
||||
@@ -385,11 +383,11 @@ class TestHa(PostgresInit):
|
||||
|
||||
def test_promoted_by_acquiring_lock(self):
|
||||
self.ha.is_healthiest_node = true
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
|
||||
def test_promotion_cancelled_after_pre_promote_failed(self):
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.p._pre_promote = false
|
||||
self.ha._is_healthiest_node = true
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
@@ -404,7 +402,7 @@ class TestHa(PostgresInit):
|
||||
@patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
|
||||
def test_long_promote(self):
|
||||
self.ha.has_lock = true
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.p.set_role('primary')
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
|
||||
@@ -415,7 +413,7 @@ class TestHa(PostgresInit):
|
||||
def test_follow_new_leader_after_failing_to_obtain_lock(self):
|
||||
self.ha.is_healthiest_node = true
|
||||
self.ha.acquire_lock = false
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.assertEqual(self.ha.run_cycle(), 'following new leader after trying and failing to obtain lock')
|
||||
|
||||
def test_demote_because_not_healthiest(self):
|
||||
@@ -424,21 +422,21 @@ class TestHa(PostgresInit):
|
||||
|
||||
def test_follow_new_leader_because_not_healthiest(self):
|
||||
self.ha.is_healthiest_node = false
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
|
||||
@patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
|
||||
def test_promote_because_have_lock(self):
|
||||
self.ha.has_lock = true
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader because I had the session lock')
|
||||
|
||||
def test_promote_without_watchdog(self):
|
||||
self.ha.has_lock = true
|
||||
self.p.is_primary = true
|
||||
self.p.is_leader = true
|
||||
with patch.object(Watchdog, 'activate', Mock(return_value=False)):
|
||||
self.assertEqual(self.ha.run_cycle(), 'Demoting self because watchdog could not be activated')
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.assertEqual(self.ha.run_cycle(), 'Not promoting self because watchdog could not be activated')
|
||||
|
||||
def test_leader_with_lock(self):
|
||||
@@ -464,12 +462,12 @@ class TestHa(PostgresInit):
|
||||
self.assertEqual(self.ha.run_cycle(), 'demoted self because failed to update leader lock in DCS')
|
||||
with patch.object(Ha, '_get_node_to_follow', Mock(side_effect=DCSError('foo'))):
|
||||
self.assertEqual(self.ha.run_cycle(), 'demoted self because failed to update leader lock in DCS')
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.assertEqual(self.ha.run_cycle(), 'not promoting because failed to update leader lock in DCS')
|
||||
|
||||
@patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
|
||||
def test_follow(self):
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), a secondary, and following a leader ()')
|
||||
self.ha.patroni.replicatefrom = "foo"
|
||||
self.p.config.check_recovery_conf = Mock(return_value=(True, False))
|
||||
@@ -486,13 +484,13 @@ class TestHa(PostgresInit):
|
||||
def test_follow_in_pause(self):
|
||||
self.ha.is_paused = true
|
||||
self.assertEqual(self.ha.run_cycle(), 'PAUSE: continue to run as primary without lock')
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.assertEqual(self.ha.run_cycle(), 'PAUSE: no action. I am (postgresql0)')
|
||||
|
||||
@patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
|
||||
@patch.object(Rewind, 'can_rewind', PropertyMock(return_value=True))
|
||||
def test_follow_triggers_rewind(self):
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.ha._rewind.trigger_check_diverged_lsn()
|
||||
self.ha.cluster = get_cluster_initialized_with_leader()
|
||||
self.assertEqual(self.ha.run_cycle(), 'running pg_rewind from leader')
|
||||
@@ -546,7 +544,7 @@ class TestHa(PostgresInit):
|
||||
self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
|
||||
self.ha.update_failsafe({'name': 'leader', 'api_url': 'http://127.0.0.1:8008/patroni',
|
||||
'conn_url': 'postgres://127.0.0.1:5432/postgres', 'slots': {'foo': 1000}})
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.assertEqual(self.ha.run_cycle(), 'DCS is not accessible')
|
||||
|
||||
def test_no_dcs_connection_replica_failsafe_not_enabled_but_active(self):
|
||||
@@ -554,7 +552,7 @@ class TestHa(PostgresInit):
|
||||
self.ha.cluster = get_cluster_initialized_with_leader()
|
||||
self.ha.update_failsafe({'name': 'leader', 'api_url': 'http://127.0.0.1:8008/patroni',
|
||||
'conn_url': 'postgres://127.0.0.1:5432/postgres', 'slots': {'foo': 1000}})
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.assertEqual(self.ha.run_cycle(), 'DCS is not accessible')
|
||||
|
||||
def test_update_failsafe(self):
|
||||
@@ -593,9 +591,9 @@ class TestHa(PostgresInit):
|
||||
self.ha.cluster = get_cluster_not_initialized_without_leader()
|
||||
self.e.initialize = true
|
||||
self.assertEqual(self.ha.bootstrap(), 'trying to bootstrap a new cluster')
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.assertEqual(self.ha.run_cycle(), 'waiting for end of recovery after bootstrap')
|
||||
self.p.is_primary = true
|
||||
self.p.is_leader = true
|
||||
self.ha.is_synchronous_mode = true
|
||||
self.assertEqual(self.ha.run_cycle(), 'running post_bootstrap')
|
||||
self.assertEqual(self.ha.run_cycle(), 'initialized a new cluster')
|
||||
@@ -615,7 +613,7 @@ class TestHa(PostgresInit):
|
||||
self.ha.cluster = get_cluster_not_initialized_without_leader()
|
||||
self.e.initialize = true
|
||||
self.ha.bootstrap()
|
||||
self.p.is_primary = true
|
||||
self.p.is_leader = true
|
||||
with patch.object(Watchdog, 'activate', Mock(return_value=False)), \
|
||||
patch('patroni.ha.logger.error') as mock_logger:
|
||||
self.assertEqual(self.ha.post_bootstrap(), 'running post_bootstrap')
|
||||
@@ -689,289 +687,112 @@ class TestHa(PostgresInit):
|
||||
|
||||
@patch('patroni.postgresql.citus.CitusHandler.is_coordinator', Mock(return_value=False))
|
||||
def test_manual_failover_from_leader(self):
|
||||
self.ha.has_lock = true # I am the leader
|
||||
|
||||
# to me
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
mock_warning.assert_called_with('%s: I am already the leader, no need to %s', 'manual failover', 'failover')
|
||||
|
||||
# to a non-existent candidate
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
mock_warning.assert_called_with(
|
||||
'%s: no healthy members found, %s is not possible', 'manual failover', 'failover')
|
||||
|
||||
# to an existent candidate
|
||||
self.ha.fetch_node_status = get_node_status()
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'b', None))
|
||||
self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
|
||||
self.ha.has_lock = true
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
f = Failover(0, self.p.name, '', None)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(f)
|
||||
self.assertEqual(self.ha.run_cycle(), 'manual failover: demoting myself')
|
||||
|
||||
# to a candidate on an older timeline
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.fetch_node_status = get_node_status(timeline=1)
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.assertEqual(mock_info.call_args_list[0][0],
|
||||
('Timeline %s of member %s is behind the cluster timeline %s', 1, 'b', 2))
|
||||
|
||||
# to a lagging candidate
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.fetch_node_status = get_node_status(wal_position=1)
|
||||
self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5})
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.assertEqual(mock_info.call_args_list[0][0],
|
||||
('Member %s exceeds maximum replication lag', 'b'))
|
||||
self.ha.cluster.members.pop()
|
||||
|
||||
@patch('patroni.postgresql.citus.CitusHandler.is_coordinator', Mock(return_value=False))
|
||||
def test_manual_switchover_from_leader(self):
|
||||
self.ha.has_lock = true # I am the leader
|
||||
|
||||
self.ha.fetch_node_status = get_node_status()
|
||||
|
||||
# different leader specified in failover key, no candidate
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
mock_warning.assert_called_with(
|
||||
'%s: leader name does not match: %s != %s', 'switchover', 'blabla', 'postgresql0')
|
||||
|
||||
# no candidate
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, '', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'switchover: demoting myself')
|
||||
|
||||
self.ha._rewind.rewind_or_reinitialize_needed_and_possible = true
|
||||
self.assertEqual(self.ha.run_cycle(), 'switchover: demoting myself')
|
||||
self.assertEqual(self.ha.run_cycle(), 'manual failover: demoting myself')
|
||||
self.ha.fetch_node_status = get_node_status(nofailover=True)
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.ha.fetch_node_status = get_node_status(watchdog_failed=True)
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.ha.fetch_node_status = get_node_status(timeline=1)
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.ha.fetch_node_status = get_node_status(wal_position=1)
|
||||
self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5})
|
||||
self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
# manual failover from the previous leader to us won't happen if we hold the nofailover flag
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
|
||||
# other members with failover_limitation_s
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.fetch_node_status = get_node_status(nofailover=True)
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.assertEqual(mock_info.call_args_list[0][0], ('Member %s is %s', 'leader', 'not allowed to promote'))
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.fetch_node_status = get_node_status(watchdog_failed=True)
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.assertEqual(mock_info.call_args_list[0][0], ('Member %s is %s', 'leader', 'not watchdog capable'))
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.fetch_node_status = get_node_status(timeline=1)
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.assertEqual(mock_info.call_args_list[0][0],
|
||||
('Timeline %s of member %s is behind the cluster timeline %s', 1, 'leader', 2))
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.fetch_node_status = get_node_status(wal_position=1)
|
||||
self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5})
|
||||
self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.assertEqual(mock_info.call_args_list[0][0], ('Member %s exceeds maximum replication lag', 'leader'))
|
||||
# Failover scheduled time must include timezone
|
||||
scheduled = datetime.datetime.now()
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.ha.run_cycle()
|
||||
|
||||
@patch('patroni.postgresql.citus.CitusHandler.is_coordinator', Mock(return_value=False))
|
||||
def test_scheduled_switchover_from_leader(self):
|
||||
self.ha.has_lock = true # I am the leader
|
||||
|
||||
self.ha.fetch_node_status = get_node_status()
|
||||
|
||||
# switchover scheduled time must include timezone
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
scheduled = datetime.datetime.now()
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'blabla', scheduled))
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.assertIn('Incorrect value of scheduled_at: %s', mock_warning.call_args_list[0][0])
|
||||
|
||||
# scheduled now
|
||||
scheduled = datetime.datetime.utcnow().replace(tzinfo=tzutc)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'b', scheduled))
|
||||
self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
|
||||
self.assertEqual('switchover: demoting myself', self.ha.run_cycle())
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
|
||||
# scheduled in the future
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
scheduled = scheduled + datetime.timedelta(seconds=30)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'blabla', scheduled))
|
||||
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
self.assertIn('Awaiting %s at %s (in %.0f seconds)', mock_info.call_args_list[0][0])
|
||||
scheduled = scheduled + datetime.timedelta(seconds=30)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
|
||||
# stale value
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
scheduled = scheduled + datetime.timedelta(seconds=-600)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'b', scheduled))
|
||||
self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
|
||||
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
self.assertIn('Found a stale %s value, cleaning up: %s', mock_warning.call_args_list[0][0])
|
||||
scheduled = scheduled + datetime.timedelta(seconds=-600)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
|
||||
def test_manual_switchover_from_leader_in_pause(self):
|
||||
self.ha.has_lock = true # I am the leader
|
||||
self.ha.is_paused = true
|
||||
|
||||
# no candidate
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, '', None))
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
self.assertEqual('PAUSE: no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
mock_warning.assert_called_with(
|
||||
'%s is possible only to a specific candidate in a paused state', 'Switchover')
|
||||
scheduled = None
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
|
||||
def test_manual_failover_from_leader_in_pause(self):
|
||||
self.ha.has_lock = true
|
||||
self.ha.fetch_node_status = get_node_status()
|
||||
self.ha.is_paused = true
|
||||
|
||||
# failover from me, candidate is healthy
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, None, 'b', None))
|
||||
self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
|
||||
self.assertEqual('PAUSE: manual failover: demoting myself', self.ha.run_cycle())
|
||||
self.ha.cluster.members.pop()
|
||||
scheduled = datetime.datetime.now()
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.assertEqual('PAUSE: no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, '', None))
|
||||
self.assertEqual('PAUSE: no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
|
||||
def test_manual_failover_from_leader_in_synchronous_mode(self):
|
||||
self.ha.is_synchronous_mode = true
|
||||
self.ha.process_sync_replication = Mock()
|
||||
self.ha.fetch_node_status = get_node_status()
|
||||
|
||||
# I am the leader
|
||||
self.p.is_primary = true
|
||||
self.p.is_leader = true
|
||||
self.ha.has_lock = true
|
||||
|
||||
# the candidate is not in sync members but we allow failover to an async candidate
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, None, 'b', None), sync=(self.p.name, 'a'))
|
||||
self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
|
||||
self.ha.is_synchronous_mode = true
|
||||
self.ha.is_failover_possible = false
|
||||
self.ha.process_sync_replication = Mock()
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None), (self.p.name, None))
|
||||
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None), (self.p.name, 'a'))
|
||||
self.ha.is_failover_possible = true
|
||||
self.assertEqual('manual failover: demoting myself', self.ha.run_cycle())
|
||||
self.ha.cluster.members.pop()
|
||||
|
||||
def test_manual_switchover_from_leader_in_synchronous_mode(self):
|
||||
self.ha.is_synchronous_mode = true
|
||||
self.ha.process_sync_replication = Mock()
|
||||
|
||||
# I am the leader
|
||||
self.p.is_primary = true
|
||||
self.ha.has_lock = true
|
||||
|
||||
# candidate specified is not in sync members
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None),
|
||||
sync=(self.p.name, 'blabla'))
|
||||
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
self.assertEqual(mock_warning.call_args_list[0][0],
|
||||
('%s candidate=%s does not match with sync_standbys=%s', 'Switchover', 'a', 'blabla'))
|
||||
|
||||
# the candidate is in sync members and is healthy
|
||||
self.ha.fetch_node_status = get_node_status(wal_position=305419896)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None),
|
||||
sync=(self.p.name, 'a'))
|
||||
self.ha.cluster.members.append(Member(0, 'a', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
|
||||
self.assertEqual('switchover: demoting myself', self.ha.run_cycle())
|
||||
|
||||
# the candidate is in sync members but is not healthy
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.fetch_node_status = get_node_status(nofailover=true)
|
||||
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
self.assertEqual(mock_info.call_args_list[0][0], ('Member %s is %s', 'a', 'not allowed to promote'))
|
||||
|
||||
def test_manual_failover_process_no_leader(self):
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', self.p.name, None))
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None))
|
||||
self.p.set_role('replica')
|
||||
|
||||
# failover to another member, fetch_node_status for candidate fails
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
self.assertEqual(mock_warning.call_args_list[1][0],
|
||||
('%s: member %s is %s', 'manual failover', 'leader', 'not reachable'))
|
||||
|
||||
# failover to another member, candidate is accessible, in_recovery
|
||||
self.p.set_role('replica')
|
||||
self.ha.fetch_node_status = get_node_status()
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
self.ha.fetch_node_status = get_node_status() # accessible, in_recovery
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
|
||||
# set nofailover flag to True for all members of the cluster
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, self.p.name, '', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
self.ha.fetch_node_status = get_node_status(reachable=False) # inaccessible, in_recovery
|
||||
self.p.set_role('replica')
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
# set failover flag to True for all members of the cluster
|
||||
# this should elect the current member, as we are not going to call the API for it.
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
|
||||
self.ha.fetch_node_status = get_node_status(nofailover=True)
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
|
||||
# failover to me but I am set to nofailover. In no case I should be elected as a leader
|
||||
self.ha.fetch_node_status = get_node_status(nofailover=True) # accessible, in_recovery
|
||||
self.p.set_role('replica')
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'postgresql0', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
# same as previous, but set the current member to nofailover. In no case it should be elected as a leader
|
||||
self.ha.patroni.nofailover = True
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because I am not allowed to promote')
|
||||
|
||||
self.ha.patroni.nofailover = False
|
||||
|
||||
# failover to another member that is on an older timeline (only failover_limitation() is checked)
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'b', None))
|
||||
self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
|
||||
self.ha.fetch_node_status = get_node_status(timeline=1)
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
mock_info.assert_called_with('%s: to %s, i am %s', 'manual failover', 'b', 'postgresql0')
|
||||
|
||||
# failover to another member lagging behind the cluster_lsn (only failover_limitation() is checked)
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5})
|
||||
self.ha.fetch_node_status = get_node_status(wal_position=1)
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
mock_info.assert_called_with('%s: to %s, i am %s', 'manual failover', 'b', 'postgresql0')
|
||||
|
||||
def test_manual_switchover_process_no_leader(self):
|
||||
self.p.is_primary = false
|
||||
self.p.set_role('replica')
|
||||
|
||||
# I was the leader, other members are healthy
|
||||
self.ha.fetch_node_status = get_node_status()
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, self.p.name, '', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
|
||||
# I was the leader, I am the only healthy member
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.fetch_node_status = get_node_status(reachable=False) # inaccessible, in_recovery
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
self.assertEqual(mock_info.call_args_list[0][0], ('Member %s is %s', 'leader', 'not reachable'))
|
||||
self.assertEqual(mock_info.call_args_list[1][0], ('Member %s is %s', 'other', 'not reachable'))
|
||||
|
||||
def test_manual_failover_process_no_leader_in_synchronous_mode(self):
|
||||
self.ha.is_synchronous_mode = true
|
||||
self.p.is_primary = false
|
||||
self.ha.fetch_node_status = get_node_status(nofailover=True) # other nodes are not healthy
|
||||
self.p.is_leader = false
|
||||
|
||||
# manual failover when our name (postgresql0) isn't in the /sync key and the candidate node is not available
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None),
|
||||
sync=('leader1', 'blabla'))
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
|
||||
# manual failover when the candidate node isn't available but our name is in the /sync key
|
||||
# while other sync node is nofailover
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None),
|
||||
sync=('leader1', 'postgresql0'))
|
||||
self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(), CaseInsensitiveSet()))
|
||||
self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
self.assertEqual(mock_warning.call_args_list[0][0],
|
||||
('%s: member %s is %s', 'manual failover', 'other', 'not allowed to promote'))
|
||||
|
||||
# manual failover to our node (postgresql0),
|
||||
# which name is not in sync nodes list (some sync nodes are available)
|
||||
self.p.set_role('replica')
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'postgresql0', None),
|
||||
sync=('leader1', 'other'))
|
||||
self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(['leader1']),
|
||||
CaseInsensitiveSet(['leader1'])))
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
|
||||
def test_manual_switchover_process_no_leader_in_synchronous_mode(self):
|
||||
self.ha.is_synchronous_mode = true
|
||||
self.p.is_primary = false
|
||||
|
||||
# to a specific node, which name doesn't match our name (postgresql0)
|
||||
# switchover to a specific node, which name doesn't match our name (postgresql0)
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'other', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
|
||||
# to our node (postgresql0), which name is not in sync nodes list
|
||||
# switchover to our node (postgresql0), which name is not in sync nodes list
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'postgresql0', None),
|
||||
sync=('leader1', 'blabla'))
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
|
||||
# without candidate, our name (postgresql0) is not in the sync nodes list
|
||||
# switchover from a specific leader, but our name (postgresql0) is not in the sync nodes list
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', '', None),
|
||||
sync=('leader', 'blabla'))
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
@@ -981,39 +802,53 @@ class TestHa(PostgresInit):
|
||||
sync=('postgresql0'))
|
||||
self.ha.patroni.nofailover = True
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because I am not allowed to promote')
|
||||
self.ha.patroni.nofailover = False
|
||||
|
||||
# manual failover when our name (postgresql0) isn't in the /sync key and the `other` node is not available
|
||||
self.ha.fetch_node_status = get_node_status(nofailover=True) # accessible, in_recovery
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None),
|
||||
sync=('leader1', 'blabla'))
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
|
||||
# manual failover when the `other` node isn't available but our name is in the /sync key
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None),
|
||||
sync=('leader1', 'postgresql0'))
|
||||
self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(), CaseInsensitiveSet()))
|
||||
self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
|
||||
# manual failover to our node (postgresql0),
|
||||
# which name is not in sync nodes list (the leader and all sync nodes are not available)
|
||||
self.p.set_role('replica')
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'postgresql0', None),
|
||||
sync=('leader1', 'other'))
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
|
||||
# manual failover to our node (postgresql0),
|
||||
# which name is not in sync nodes list (some sync nodes are available)
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'postgresql0', None),
|
||||
sync=('leader1', 'other'))
|
||||
self.p.set_role('replica')
|
||||
self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(['leader1']),
|
||||
CaseInsensitiveSet(['leader1'])))
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
|
||||
def test_manual_failover_process_no_leader_in_pause(self):
|
||||
self.ha.is_paused = true
|
||||
|
||||
# I am running as primary, cluster is unlocked, the candidate is allowed to promote
|
||||
# but we are in pause
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'PAUSE: continue to run as primary without lock')
|
||||
|
||||
def test_manual_switchover_process_no_leader_in_pause(self):
|
||||
self.ha.is_paused = true
|
||||
|
||||
# I am running as primary, cluster is unlocked, no candidate specified
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', '', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'PAUSE: continue to run as primary without lock')
|
||||
|
||||
# the candidate is not running
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'blabla', None))
|
||||
self.assertEqual('PAUSE: acquired session lock as a leader', self.ha.run_cycle())
|
||||
self.assertEqual(
|
||||
mock_warning.call_args_list[0][0],
|
||||
('%s: removing failover key because failover candidate is not running', 'switchover'))
|
||||
|
||||
# switchover to me, I am not leader
|
||||
self.p.is_primary = false
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'blabla', None))
|
||||
self.assertEqual('PAUSE: acquired session lock as a leader', self.ha.run_cycle())
|
||||
self.p.is_leader = false
|
||||
self.p.set_role('replica')
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', self.p.name, None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'PAUSE: promoted self to leader by acquiring session lock')
|
||||
|
||||
def test_is_healthiest_node(self):
|
||||
self.ha.is_failsafe_mode = true
|
||||
self.ha.state_handler.is_primary = false
|
||||
self.ha.state_handler.is_leader = false
|
||||
self.ha.patroni.nofailover = False
|
||||
self.ha.fetch_node_status = get_node_status()
|
||||
self.ha.dcs._last_failsafe = {'foo': ''}
|
||||
@@ -1027,7 +862,7 @@ class TestHa(PostgresInit):
|
||||
self.assertFalse(self.ha.is_healthiest_node())
|
||||
|
||||
def test__is_healthiest_node(self):
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(sync=('postgresql1', self.p.name))
|
||||
self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
|
||||
self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
|
||||
@@ -1126,7 +961,7 @@ class TestHa(PostgresInit):
|
||||
self.assertTrue(self.ha.restart_matches("replica", "9.5.2", False))
|
||||
|
||||
def test_process_healthy_cluster_in_pause(self):
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.ha.is_paused = true
|
||||
self.p.name = 'leader'
|
||||
self.ha.cluster = get_cluster_initialized_with_leader()
|
||||
@@ -1137,7 +972,7 @@ class TestHa(PostgresInit):
|
||||
@patch('patroni.postgresql.mtime', Mock(return_value=1588316884))
|
||||
@patch('builtins.open', mock_open(read_data='1\t0/40159C0\tno recovery target specified\n'))
|
||||
def test_process_healthy_standby_cluster_as_standby_leader(self):
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.p.name = 'leader'
|
||||
self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
|
||||
self.p.config.check_recovery_conf = Mock(return_value=(False, False))
|
||||
@@ -1149,7 +984,7 @@ class TestHa(PostgresInit):
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to a standby leader because i had the session lock')
|
||||
|
||||
def test_process_healthy_standby_cluster_as_cascade_replica(self):
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.p.name = 'replica'
|
||||
self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
|
||||
self.assertEqual(self.ha.run_cycle(),
|
||||
@@ -1159,7 +994,7 @@ class TestHa(PostgresInit):
|
||||
|
||||
@patch.object(Cluster, 'is_unlocked', Mock(return_value=True))
|
||||
def test_process_unhealthy_standby_cluster_as_standby_leader(self):
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.p.name = 'leader'
|
||||
self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
|
||||
self.ha.sysid_valid = true
|
||||
@@ -1169,13 +1004,13 @@ class TestHa(PostgresInit):
|
||||
@patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
|
||||
@patch.object(Rewind, 'can_rewind', PropertyMock(return_value=True))
|
||||
def test_process_unhealthy_standby_cluster_as_cascade_replica(self):
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.p.name = 'replica'
|
||||
self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
|
||||
self.assertTrue(self.ha.run_cycle().startswith('running pg_rewind from remote_member:'))
|
||||
|
||||
def test_recover_unhealthy_leader_in_standby_cluster(self):
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.p.name = 'leader'
|
||||
self.p.is_running = false
|
||||
self.p.follow = false
|
||||
@@ -1184,7 +1019,7 @@ class TestHa(PostgresInit):
|
||||
|
||||
@patch.object(Cluster, 'is_unlocked', Mock(return_value=True))
|
||||
def test_recover_unhealthy_unlocked_standby_cluster(self):
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.p.name = 'leader'
|
||||
self.p.is_running = false
|
||||
self.p.follow = false
|
||||
@@ -1244,7 +1079,7 @@ class TestHa(PostgresInit):
|
||||
check_calls([(update_lock, True), (demote, True)])
|
||||
|
||||
self.ha.has_lock = false
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.assertEqual(self.ha.run_cycle(),
|
||||
'no action. I am (postgresql0), a secondary, and following a leader (leader)')
|
||||
check_calls([(update_lock, False), (demote, False)])
|
||||
@@ -1255,7 +1090,7 @@ class TestHa(PostgresInit):
|
||||
f = Failover(0, self.p.name, '', None)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(f)
|
||||
self.ha.fetch_node_status = get_node_status() # accessible, in_recovery
|
||||
self.assertEqual(self.ha.run_cycle(), 'switchover: demoting myself')
|
||||
self.assertEqual(self.ha.run_cycle(), 'manual failover: demoting myself')
|
||||
|
||||
@patch('patroni.ha.Ha.demote')
|
||||
def test_failover_immediately_on_zero_primary_start_timeout(self, demote):
|
||||
@@ -1378,7 +1213,7 @@ class TestHa(PostgresInit):
|
||||
self.ha.is_synchronous_mode = true
|
||||
|
||||
mock_set_sync = self.p.sync_handler.set_synchronous_standby_names = Mock()
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.p.set_role('replica')
|
||||
self.ha.has_lock = true
|
||||
mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
|
||||
@@ -1401,7 +1236,7 @@ class TestHa(PostgresInit):
|
||||
def test_unhealthy_sync_mode(self):
|
||||
self.ha.is_synchronous_mode = true
|
||||
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.p.set_role('replica')
|
||||
self.p.name = 'other'
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(sync=('leader', 'other2'))
|
||||
@@ -1432,7 +1267,7 @@ class TestHa(PostgresInit):
|
||||
self.ha.is_synchronous_mode = true
|
||||
|
||||
self.p.name = 'other'
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.p.set_role('replica')
|
||||
mock_restart = self.p.restart = Mock(return_value=True)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
|
||||
@@ -1550,7 +1385,7 @@ class TestHa(PostgresInit):
|
||||
@patch('sys.exit', return_value=1)
|
||||
def test_abort_join(self, exit_mock):
|
||||
self.ha.cluster = get_cluster_not_initialized_without_leader()
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.ha.run_cycle()
|
||||
exit_mock.assert_called_once_with(1)
|
||||
|
||||
@@ -1610,7 +1445,7 @@ class TestHa(PostgresInit):
|
||||
@patch.object(SlotsHandler, 'sync_replication_slots', Mock(return_value=['ls']))
|
||||
def test_follow_copy(self):
|
||||
self.ha.cluster.config.data['slots'] = {'ls': {'database': 'a', 'plugin': 'b'}}
|
||||
self.p.is_primary = false
|
||||
self.p.is_leader = false
|
||||
self.assertTrue(self.ha.run_cycle().startswith('Copying logical slots'))
|
||||
|
||||
def test_acquire_lock(self):
|
||||
@@ -1630,13 +1465,3 @@ class TestHa(PostgresInit):
|
||||
self.assertEqual(self.ha.patroni.request.call_args[1]['timeout'], 2)
|
||||
mock_logger.assert_called()
|
||||
self.assertTrue(mock_logger.call_args[0][0].startswith('Request to Citus coordinator'))
|
||||
|
||||
def test_has_members_eligible_to_promote(self):
|
||||
self.ha.fetch_node_status = get_node_status()
|
||||
members = [
|
||||
Member(0, 'test', 1, {'api_url': 'http://127.0.0.1:8011/patroni', 'conn_url': 'postgres://127.0.0.1:5432/postgres'}),
|
||||
Member(0, 'test2', 1, {'api_url': 'http://127.0.0.1:8011/patroni', 'conn_url': 'postgres://127.0.0.1:5432/postgres'}),
|
||||
]
|
||||
with patch('patroni.ha.logger.info') as mock_logger:
|
||||
self.assertTrue(self.ha.has_members_eligible_to_promote(members, fast_path=True))
|
||||
mock_logger.assert_not_called()
|
||||
|
||||
@@ -326,7 +326,7 @@ class TestKubernetesConfigMaps(BaseTestKubernetes):
|
||||
self.k.initialize()
|
||||
|
||||
def test_delete_leader(self):
|
||||
self.k.delete_leader(self.k.get_cluster().leader, 1)
|
||||
self.k.delete_leader(1)
|
||||
|
||||
def test_cancel_initialization(self):
|
||||
self.k.cancel_initialization()
|
||||
|
||||
@@ -185,7 +185,7 @@ class TestPatroni(unittest.TestCase):
|
||||
|
||||
def test_reload_config(self):
|
||||
self.p.reload_config()
|
||||
self.p._get_tags = Mock(side_effect=Exception)
|
||||
self.p.get_tags = Mock(side_effect=Exception)
|
||||
self.p.reload_config(local=True)
|
||||
|
||||
def test_nosync(self):
|
||||
|
||||
+12
-25
@@ -374,11 +374,11 @@ class TestPostgresql(BaseTestPostgresql):
|
||||
self.assertRaises(psycopg.ProgrammingError, self.p.query, 'blabla')
|
||||
|
||||
@patch.object(Postgresql, 'pg_isready', Mock(return_value=STATE_REJECT))
|
||||
def test_is_primary(self):
|
||||
self.assertTrue(self.p.is_primary())
|
||||
def test_is_leader(self):
|
||||
self.assertTrue(self.p.is_leader())
|
||||
self.p.reset_cluster_info_state(None)
|
||||
with patch.object(Postgresql, '_query', Mock(side_effect=RetryFailedError(''))):
|
||||
self.assertFalse(self.p.is_primary())
|
||||
self.assertFalse(self.p.is_leader())
|
||||
|
||||
@patch.object(Postgresql, 'controldata', Mock(return_value={'Database cluster state': 'shut down',
|
||||
'Latest checkpoint location': '0/1ADBC18',
|
||||
@@ -472,7 +472,7 @@ class TestPostgresql(BaseTestPostgresql):
|
||||
self.assertIsNone(self.p.call_nowait(CallbackAction.ON_START))
|
||||
|
||||
@patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
|
||||
def test_is_primary_exception(self):
|
||||
def test_is_leader_exception(self):
|
||||
self.p.start()
|
||||
self.p.query = Mock(side_effect=psycopg.OperationalError("not supported"))
|
||||
self.assertTrue(self.p.stop())
|
||||
@@ -556,7 +556,9 @@ class TestPostgresql(BaseTestPostgresql):
|
||||
|
||||
@patch('time.sleep', Mock())
|
||||
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
|
||||
def test_reload_config(self):
|
||||
@patch.object(MockCursor, 'fetchone')
|
||||
def test_reload_config(self, mock_fetchone):
|
||||
mock_fetchone.return_value = (1,)
|
||||
parameters = self._PARAMETERS.copy()
|
||||
parameters.pop('f.oo')
|
||||
parameters['wal_buffers'] = '512'
|
||||
@@ -564,14 +566,9 @@ class TestPostgresql(BaseTestPostgresql):
|
||||
'authentication': {},
|
||||
'retry_timeout': 10, 'listen': '*', 'krbsrvname': 'postgres', 'parameters': parameters}
|
||||
self.p.reload_config(config)
|
||||
mock_fetchone.side_effect = Exception
|
||||
parameters['b.ar'] = 'bar'
|
||||
with patch.object(MockCursor, 'fetchall',
|
||||
Mock(side_effect=[[('wal_block_size', '8191', None, 'integer', 'internal'),
|
||||
('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
|
||||
('shared_buffers', '16384', '8kB', 'integer', 'postmaster'),
|
||||
('wal_buffers', '-1', '8kB', 'integer', 'postmaster'),
|
||||
('port', '5433', None, 'integer', 'postmaster')], Exception])):
|
||||
self.p.reload_config(config)
|
||||
self.p.reload_config(config)
|
||||
parameters['autovacuum'] = 'on'
|
||||
self.p.reload_config(config)
|
||||
parameters['autovacuum'] = 'off'
|
||||
@@ -588,10 +585,7 @@ class TestPostgresql(BaseTestPostgresql):
|
||||
self.assertEqual(self.p.config.local_replication_address, {'host': '/tmp', 'port': '5432'})
|
||||
self.p.config._server_parameters.pop('unix_socket_directories')
|
||||
self.p.config.resolve_connection_addresses()
|
||||
self.assertEqual(self.p.connection_pool.conn_kwargs, {'connect_timeout': 3, 'dbname': 'postgres',
|
||||
'fallback_application_name': 'Patroni',
|
||||
'options': '-c statement_timeout=2000',
|
||||
'password': 'test', 'port': '5432', 'user': 'foo'})
|
||||
self.assertEqual(self.p.config._local_address, {'port': '5432'})
|
||||
|
||||
@patch.object(Postgresql, '_version_file_exists', Mock(return_value=True))
|
||||
def test_get_major_version(self):
|
||||
@@ -602,7 +596,7 @@ class TestPostgresql(BaseTestPostgresql):
|
||||
|
||||
def test_postmaster_start_time(self):
|
||||
now = datetime.datetime.now()
|
||||
with patch.object(MockCursor, "fetchall", Mock(return_value=[(now, True, '', '', '', '', False)])):
|
||||
with patch.object(MockCursor, "fetchone", Mock(return_value=(now, True, '', '', '', '', False))):
|
||||
self.assertEqual(self.p.postmaster_start_time(), now.isoformat(sep=' '))
|
||||
t = Thread(target=self.p.postmaster_start_time)
|
||||
t.start()
|
||||
@@ -688,7 +682,7 @@ class TestPostgresql(BaseTestPostgresql):
|
||||
self.assertIsNone(self.p.wait_for_startup())
|
||||
|
||||
def test_get_server_parameters(self):
|
||||
config = {'parameters': {'wal_level': 'hot_standby'}, 'listen': '0'}
|
||||
config = {'parameters': {'wal_level': 'hot_standby', 'max_prepared_transactions': 100}, 'listen': '0'}
|
||||
self.p._global_config = GlobalConfig({'synchronous_mode': True})
|
||||
self.p.config.get_server_parameters(config)
|
||||
self.p._global_config = GlobalConfig({'synchronous_mode': True, 'synchronous_mode_strict': True})
|
||||
@@ -972,10 +966,3 @@ class TestPostgresql2(BaseTestPostgresql):
|
||||
gucs = self.p.available_gucs
|
||||
self.assertIsInstance(gucs, CaseInsensitiveSet)
|
||||
self.assertEqual(gucs, mock_available_gucs.return_value)
|
||||
|
||||
def test_cluster_info_query(self):
|
||||
self.assertIn('diff(pg_catalog.pg_current_wal_flush_lsn(', self.p.cluster_info_query)
|
||||
self.p._major_version = 90600
|
||||
self.assertIn('diff(pg_catalog.pg_current_xlog_flush_location(', self.p.cluster_info_query)
|
||||
self.p._major_version = 90500
|
||||
self.assertIn('diff(pg_catalog.pg_current_xlog_location(', self.p.cluster_info_query)
|
||||
|
||||
+4
-4
@@ -142,25 +142,25 @@ class TestRaft(unittest.TestCase):
|
||||
raft._citus_group = '1'
|
||||
self.assertTrue(raft.manual_failover('foo', 'bar'))
|
||||
raft._citus_group = '0'
|
||||
self.assertTrue(raft.take_leader())
|
||||
cluster = raft.get_cluster()
|
||||
self.assertIsInstance(cluster, Cluster)
|
||||
self.assertIsInstance(cluster.workers[1], Cluster)
|
||||
leader = cluster.leader
|
||||
self.assertTrue(raft.delete_leader(leader))
|
||||
self.assertTrue(raft._sync_obj.set(raft.status_path, '{"optime":1234567,"slots":{"ls":12345}}'))
|
||||
raft.get_cluster()
|
||||
leader = raft.get_cluster().leader
|
||||
self.assertTrue(raft.update_leader(leader, '1', failsafe={'foo': 'bat'}))
|
||||
self.assertTrue(raft._sync_obj.set(raft.failsafe_path, '{"foo"}'))
|
||||
self.assertTrue(raft._sync_obj.set(raft.status_path, '{'))
|
||||
raft.get_citus_coordinator()
|
||||
self.assertTrue(raft.delete_sync_state())
|
||||
self.assertTrue(raft.delete_leader())
|
||||
self.assertTrue(raft.set_history_value(''))
|
||||
self.assertTrue(raft.delete_cluster())
|
||||
raft._citus_group = '1'
|
||||
self.assertTrue(raft.delete_cluster())
|
||||
raft._citus_group = None
|
||||
raft.get_cluster()
|
||||
self.assertTrue(raft.take_leader())
|
||||
raft.get_cluster()
|
||||
raft.watch(None, 0.001)
|
||||
raft._sync_obj.destroy()
|
||||
|
||||
|
||||
@@ -92,9 +92,8 @@ class TestRewind(BaseTestPostgresql):
|
||||
self.r.rewind_or_reinitialize_needed_and_possible(self.leader)
|
||||
|
||||
with patch.object(Postgresql, 'is_running', Mock(return_value=True)), \
|
||||
patch.object(MockCursor, 'fetchone', Mock(side_effect=Exception)), \
|
||||
patch.object(MockCursor, 'fetchall',
|
||||
Mock(return_value=[(0, 0, 1, 1, 0, 0, 0, 0, 0, None, None, None)])):
|
||||
patch.object(MockCursor, 'fetchone',
|
||||
Mock(side_effect=[Exception, (0, 0, 1, 1, 0, 0, 0, 0, 0, None, None, None)])):
|
||||
self.r.rewind_or_reinitialize_needed_and_possible(self.leader)
|
||||
|
||||
@patch.object(CancellableSubprocess, 'call', mock_cancellable_call)
|
||||
|
||||
+13
-14
@@ -32,9 +32,9 @@ class TestSlotsHandler(BaseTestPostgresql):
|
||||
self.p._global_config = GlobalConfig({})
|
||||
self.s = self.p.slots_handler
|
||||
self.p.start()
|
||||
config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}, 'ls2': None}}, 1)
|
||||
config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}}}, 1)
|
||||
self.cluster = Cluster(True, config, self.leader, 0, [self.me, self.other, self.leadermem],
|
||||
None, SyncState.empty(), None, {'ls': 12345, 'ls2': 12345}, None)
|
||||
None, SyncState.empty(), None, {'ls': 12345}, None)
|
||||
|
||||
def test_sync_replication_slots(self):
|
||||
config = ClusterConfig(1, {'slots': {'test_3': {'database': 'a', 'plugin': 'b'},
|
||||
@@ -51,7 +51,7 @@ class TestSlotsHandler(BaseTestPostgresql):
|
||||
self.s.sync_replication_slots(cluster, False)
|
||||
mock_debug.assert_called_once()
|
||||
self.p.set_role('replica')
|
||||
with patch.object(Postgresql, 'is_primary', Mock(return_value=False)), \
|
||||
with patch.object(Postgresql, 'is_leader', Mock(return_value=False)), \
|
||||
patch.object(SlotsHandler, 'drop_replication_slot') as mock_drop:
|
||||
self.s.sync_replication_slots(cluster, False, paused=True)
|
||||
mock_drop.assert_not_called()
|
||||
@@ -82,7 +82,7 @@ class TestSlotsHandler(BaseTestPostgresql):
|
||||
None, SyncState.empty(), None, {'ls': 10}, None)
|
||||
self.p.set_role('replica')
|
||||
with patch.object(Postgresql, '_query') as mock_query, \
|
||||
patch.object(Postgresql, 'is_primary', Mock(return_value=False)):
|
||||
patch.object(Postgresql, 'is_leader', Mock(return_value=False)):
|
||||
mock_query.return_value = [('ls', 'logical', 'b', 'a', 5, 12345, 105)]
|
||||
ret = self.s.sync_replication_slots(cluster, False)
|
||||
self.assertEqual(ret, [])
|
||||
@@ -96,20 +96,20 @@ class TestSlotsHandler(BaseTestPostgresql):
|
||||
self.s.sync_replication_slots(cluster, False)
|
||||
with patch.object(Postgresql, '_query') as mock_query:
|
||||
self.p.reset_cluster_info_state(None)
|
||||
mock_query.return_value = [(
|
||||
mock_query.return_value.fetchone.return_value = (
|
||||
1, 0, 0, 0, 0, 0, 0, 0, 0, None, None,
|
||||
[{"slot_name": "ls", "type": "logical", "datoid": 5, "plugin": "b",
|
||||
"confirmed_flush_lsn": 12345, "catalog_xmin": 105}])]
|
||||
"confirmed_flush_lsn": 12345, "catalog_xmin": 105}])
|
||||
self.assertEqual(self.p.slots(), {'ls': 12345})
|
||||
|
||||
self.p.reset_cluster_info_state(None)
|
||||
mock_query.return_value = [(
|
||||
mock_query.return_value.fetchone.return_value = (
|
||||
1, 0, 0, 0, 0, 0, 0, 0, 0, None, None,
|
||||
[{"slot_name": "ls", "type": "logical", "datoid": 6, "plugin": "b",
|
||||
"confirmed_flush_lsn": 12345, "catalog_xmin": 105}])]
|
||||
"confirmed_flush_lsn": 12345, "catalog_xmin": 105}])
|
||||
self.assertEqual(self.p.slots(), {})
|
||||
|
||||
@patch.object(Postgresql, 'is_primary', Mock(return_value=False))
|
||||
@patch.object(Postgresql, 'is_leader', Mock(return_value=False))
|
||||
def test__ensure_logical_slots_replica(self):
|
||||
self.p.set_role('replica')
|
||||
self.cluster.slots['ls'] = 12346
|
||||
@@ -123,7 +123,6 @@ class TestSlotsHandler(BaseTestPostgresql):
|
||||
self.assertEqual(self.s.sync_replication_slots(self.cluster, False), ['ls'])
|
||||
self.cluster.slots['ls'] = 'a'
|
||||
self.assertEqual(self.s.sync_replication_slots(self.cluster, False), [])
|
||||
self.cluster.config.data['slots']['ls']['database'] = 'b'
|
||||
with patch.object(MockCursor, 'rowcount', PropertyMock(return_value=1), create=True):
|
||||
self.assertEqual(self.s.sync_replication_slots(self.cluster, False), ['ls'])
|
||||
|
||||
@@ -137,21 +136,21 @@ class TestSlotsHandler(BaseTestPostgresql):
|
||||
|
||||
@patch.object(Postgresql, 'stop', Mock(return_value=True))
|
||||
@patch.object(Postgresql, 'start', Mock(return_value=True))
|
||||
@patch.object(Postgresql, 'is_primary', Mock(return_value=False))
|
||||
@patch.object(Postgresql, 'is_leader', Mock(return_value=False))
|
||||
def test_check_logical_slots_readiness(self):
|
||||
self.s.copy_logical_slots(self.cluster, ['ls'])
|
||||
with patch.object(MockCursor, '__iter__', Mock(return_value=iter([('postgresql0', None)]))), \
|
||||
patch.object(MockCursor, 'fetchall', Mock(side_effect=Exception)):
|
||||
patch.object(MockCursor, 'fetchone', Mock(side_effect=Exception)):
|
||||
self.assertFalse(self.s.check_logical_slots_readiness(self.cluster, None))
|
||||
with patch.object(MockCursor, '__iter__', Mock(return_value=iter([('postgresql0', None)]))), \
|
||||
patch.object(MockCursor, 'fetchall', Mock(return_value=[(False,)])):
|
||||
patch.object(MockCursor, 'fetchone', Mock(return_value=(False,))):
|
||||
self.assertFalse(self.s.check_logical_slots_readiness(self.cluster, None))
|
||||
with patch.object(MockCursor, '__iter__', Mock(return_value=iter([('ls', 100)]))):
|
||||
self.s.check_logical_slots_readiness(self.cluster, None)
|
||||
|
||||
@patch.object(Postgresql, 'stop', Mock(return_value=True))
|
||||
@patch.object(Postgresql, 'start', Mock(return_value=True))
|
||||
@patch.object(Postgresql, 'is_primary', Mock(return_value=False))
|
||||
@patch.object(Postgresql, 'is_leader', Mock(return_value=False))
|
||||
def test_on_promote(self):
|
||||
self.s.schedule_advance_slots({'foo': {'bar': 100}})
|
||||
self.s.copy_logical_slots(self.cluster, ['ls'])
|
||||
|
||||
@@ -202,7 +202,7 @@ class TestZooKeeper(unittest.TestCase):
|
||||
mock_logger.assert_called_once()
|
||||
|
||||
def test_delete_leader(self):
|
||||
self.assertTrue(self.zk.delete_leader(self.zk.get_cluster().leader))
|
||||
self.assertTrue(self.zk.delete_leader())
|
||||
|
||||
def test_set_failover_value(self):
|
||||
self.zk.set_failover_value('')
|
||||
|
||||
@@ -6,6 +6,7 @@ postgres_matrix =
|
||||
pg13: PG_MAJOR = 13
|
||||
pg14: PG_MAJOR = 14
|
||||
pg15: PG_MAJOR = 15
|
||||
pg16: PG_MAJOR = 16
|
||||
psycopg_deps =
|
||||
py{37,38,39,310,311}-{lin,win}: psycopg[binary]
|
||||
mac: psycopg2-binary
|
||||
@@ -106,7 +107,7 @@ description = Reformat code with black
|
||||
deps = black
|
||||
commands = black {posargs:patroni tests}
|
||||
|
||||
[testenv:pg{12,13,14,15}-docker-build]
|
||||
[testenv:pg{12,13,14,15,16}-docker-build]
|
||||
description = Build docker containers needed for testing
|
||||
labels =
|
||||
behave
|
||||
@@ -124,7 +125,7 @@ commands =
|
||||
--file features/Dockerfile
|
||||
allowlist_externals = docker
|
||||
|
||||
[testenv:pg{12,13,14,15}-docker-behave-{etcd}-{lin,mac}]
|
||||
[testenv:pg{12,13,14,15,16}-docker-behave-{etcd}-{lin,mac}]
|
||||
description = Run behaviour tests in patroni-dev docker container
|
||||
setenv =
|
||||
etcd: DCS=etcd
|
||||
@@ -133,7 +134,7 @@ setenv =
|
||||
labels =
|
||||
behave
|
||||
depends =
|
||||
pg{11,12,13,14,15}-docker-build
|
||||
pg{11,12,13,14,15,16}-docker-build
|
||||
|
||||
# There's a bug which affects calling multiple envs on the command line
|
||||
# This should be a valid command: tox -e 'py{36,37,38,39,310,311}-behave-{env:DCS}-lin'
|
||||
|
||||
Reference in New Issue
Block a user