From a3b3e1bc1c29d6e9a3219e33aed9e9ec89b84ac6 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 26 Sep 2023 12:30:27 +0200 Subject: [PATCH 01/11] Release v3.1.2 (#2885) - bump version - update release notes --- docs/releases.rst | 22 ++++++++++++++++++++++ patroni/version.py | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/releases.rst b/docs/releases.rst index 5d4af2cc..34c44047 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -3,6 +3,28 @@ Release notes ============= +Version 3.1.2 +------------- + +**Bugfixes** + +- Fixed bug with ``wal_keep_size`` checks (Alexander Kukushkin) + + The ``wal_keep_size`` is a GUC that normally has a unit and Patroni was failing to cast its value to ``int``. As a result the value of ``bootstrap.dcs`` was not written to the ``/config`` key afterwards. + +- Detect and resolve inconsistencies between ``/sync`` key and ``synchronous_standby_names`` (Alexander Kukushkin) + + Normally, Patroni updates ``/sync`` and ``synchronous_standby_names`` in a very specific order, but in case of a bug or when someone manually reset ``synchronous_standby_names``, Patroni was getting into an inconsistent state. As a result it was possible that the failover happens to an asynchronous node. + +- Read GUC's values when joining running Postgres (Alexander Kukushkin) + + When restarted in ``pause``, Patroni was discarding the ``synchronous_standby_names`` GUC from the ``postgresql.conf``. To solve it and avoid similar issues, Patroni will read GUC's value if it is joining an already running Postgres. + +- Silenced annoying warnings when checking for node uniqueness (Alexander Kukushkin) + + ``WARNING`` messages are produced by ``urllib3`` if Patroni is quickly restarted. + + Version 3.1.1 ------------- diff --git a/patroni/version.py b/patroni/version.py index ff98d3c0..b2ab33ce 100644 --- a/patroni/version.py +++ b/patroni/version.py @@ -2,4 +2,4 @@ :var __version__: the current Patroni version. """ -__version__ = '3.1.1' +__version__ = '3.1.2' From 220cacd95f5932567856ab5ca1726d0c12e9e196 Mon Sep 17 00:00:00 2001 From: Polina Bungina <27892524+hughcapet@users.noreply.github.com> Date: Tue, 26 Sep 2023 15:52:00 +0200 Subject: [PATCH 02/11] Don't call socket functions from tests (#2886) We used to call `socket` module's functions from the config_generator tests to later compare with the output produced by --generate-config. That however sometimes ends up with the whole test module failure if gethostname() returned None. Also includes a little code deduplication (NO_VALUE_MSG imported directly from the config_generator module) and removes debug maxDiff option --- patroni/config_generator.py | 24 +++++++++++------------ tests/test_config_generator.py | 36 +++++++++++++++------------------- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/patroni/config_generator.py b/patroni/config_generator.py index 0269c49a..4bcb2819 100644 --- a/patroni/config_generator.py +++ b/patroni/config_generator.py @@ -38,7 +38,7 @@ _AUTH_ALLOWED_PARAMETERS_MAPPING = { 'gssencmode': 'PGGSSENCMODE', 'channel_binding': 'PGCHANNELBINDING' } -_NO_VALUE_MSG = '#FIXME' +NO_VALUE_MSG = '#FIXME' def get_address() -> Tuple[str, str]: @@ -50,7 +50,7 @@ def get_address() -> Tuple[str, str]: :returns: tuple consisting of the hostname returned by :func:`~socket.gethostname` and the first element in the sorted list of the addresses returned by :func:`~socket.getaddrinfo`. Sorting guarantees it will prefer IPv4. - If an exception occured, hostname and ip values are equal to :data:`~patroni.config_generator._NO_VALUE_MSG`. + If an exception occured, hostname and ip values are equal to :data:`~patroni.config_generator.NO_VALUE_MSG`. """ hostname = None try: @@ -59,7 +59,7 @@ def get_address() -> Tuple[str, str]: key=lambda x: x[0])[0][4][0] except Exception as err: logging.warning('Failed to obtain address: %r', err) - return _NO_VALUE_MSG, _NO_VALUE_MSG + return NO_VALUE_MSG, NO_VALUE_MSG class AbstractConfigGenerator(abc.ABC): @@ -88,24 +88,24 @@ class AbstractConfigGenerator(abc.ABC): """Generate a template config for further extension (e.g. in the inherited classes). :returns: dictionary with the values gathered from Patroni env, hopefully defined hostname and ip address - (otherwise set to :data:`~patroni.config_generator._NO_VALUE_MSG`), and some sane defaults. + (otherwise set to :data:`~patroni.config_generator.NO_VALUE_MSG`), and some sane defaults. """ template_config: Dict[str, Any] = { - 'scope': _NO_VALUE_MSG, + 'scope': NO_VALUE_MSG, 'name': cls._HOSTNAME, 'postgresql': { - 'data_dir': _NO_VALUE_MSG, - 'connect_address': _NO_VALUE_MSG + ':5432', - 'listen': _NO_VALUE_MSG + ':5432', + 'data_dir': NO_VALUE_MSG, + 'connect_address': NO_VALUE_MSG + ':5432', + 'listen': NO_VALUE_MSG + ':5432', 'bin_dir': '', 'authentication': { 'superuser': { 'username': 'postgres', - 'password': _NO_VALUE_MSG + 'password': NO_VALUE_MSG }, 'replication': { 'username': 'replicator', - 'password': _NO_VALUE_MSG + 'password': NO_VALUE_MSG } } }, @@ -185,7 +185,7 @@ class SampleConfigGenerator(AbstractConfigGenerator): self.config['bootstrap']['dcs']['postgresql']['use_pg_rewind'] = True if self.pg_major >= 110000: self.config['postgresql']['authentication'].setdefault( - 'rewind', {'username': 'rewind_user'}).setdefault('password', _NO_VALUE_MSG) + 'rewind', {'username': 'rewind_user'}).setdefault('password', NO_VALUE_MSG) class RunningClusterConfigGenerator(AbstractConfigGenerator): @@ -335,7 +335,7 @@ class RunningClusterConfigGenerator(AbstractConfigGenerator): getpass('Please enter the user password:') self.config['postgresql']['authentication'] = { 'superuser': su_params, - 'replication': {'username': _NO_VALUE_MSG, 'password': _NO_VALUE_MSG} + 'replication': {'username': NO_VALUE_MSG, 'password': NO_VALUE_MSG} } def _set_conf_files(self) -> None: diff --git a/tests/test_config_generator.py b/tests/test_config_generator.py index 49799c01..e702f957 100644 --- a/tests/test_config_generator.py +++ b/tests/test_config_generator.py @@ -1,6 +1,5 @@ import os import psutil -import socket import unittest from . import MockConnect, MockCursor, MockConnectionInfo @@ -9,28 +8,25 @@ from mock import MagicMock, Mock, PropertyMock, mock_open, patch from patroni.__main__ import main as _main from patroni.config import Config -from patroni.config_generator import AbstractConfigGenerator, get_address - +from patroni.config_generator import AbstractConfigGenerator, get_address, NO_VALUE_MSG from patroni.utils import patch_config from . import psycopg_connect +HOSTNAME = 'test_hostname' +IP = '1.9.8.4' + @patch('patroni.psycopg.connect', psycopg_connect) -@patch('socket.getaddrinfo', Mock(return_value=[(0, 0, 0, 0, ('1.9.8.4', 1984))])) @patch('builtins.open', MagicMock()) @patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 16.2")) @patch('psutil.Process.exe', Mock(return_value='/bin/dir/from/running/postgres')) @patch('psutil.Process.__init__', Mock(return_value=None)) +@patch.object(AbstractConfigGenerator, '_HOSTNAME', HOSTNAME) +@patch.object(AbstractConfigGenerator, '_IP', IP) class TestGenerateConfig(unittest.TestCase): - no_value_msg = '#FIXME' - _HOSTNAME = socket.gethostname() - _IP = sorted(socket.getaddrinfo(_HOSTNAME, 0, socket.AF_UNSPEC, socket.SOCK_STREAM, 0), key=lambda x: x[0])[0][4][0] - def setUp(self): - self.maxDiff = None - os.environ['PATRONI_SCOPE'] = 'scope_from_env' os.environ['PATRONI_POSTGRESQL_BIN_DIR'] = '/bin/from/env' os.environ['PATRONI_SUPERUSER_USERNAME'] = 'su_user_from_env' @@ -54,14 +50,14 @@ class TestGenerateConfig(unittest.TestCase): self.config = { 'scope': self.environ['PATRONI_SCOPE'], - 'name': self._HOSTNAME, + 'name': HOSTNAME, 'bootstrap': { 'dcs': dynamic_config }, 'postgresql': { - 'connect_address': self.no_value_msg + ':5432', - 'data_dir': self.no_value_msg, - 'listen': self.no_value_msg + ':5432', + 'connect_address': NO_VALUE_MSG + ':5432', + 'data_dir': NO_VALUE_MSG, + 'listen': NO_VALUE_MSG + ':5432', 'pg_hba': ['host all all all md5', f'host replication {self.environ["PATRONI_REPLICATION_USERNAME"]} all md5'], 'authentication': {'superuser': {'username': self.environ['PATRONI_SUPERUSER_USERNAME'], @@ -99,7 +95,7 @@ class TestGenerateConfig(unittest.TestCase): } }, 'postgresql': { - 'connect_address': f'{self._IP}:bar', + 'connect_address': f'{IP}:bar', 'listen': '6.6.6.6:1984', 'data_dir': 'data', 'bin_dir': '/bin/dir/from/running', @@ -118,8 +114,8 @@ class TestGenerateConfig(unittest.TestCase): 'sslmode': 'prefer' }, 'replication': { - 'username': self.no_value_msg, - 'password': self.no_value_msg + 'username': NO_VALUE_MSG, + 'password': NO_VALUE_MSG }, 'rewind': None }, @@ -179,7 +175,7 @@ class TestGenerateConfig(unittest.TestCase): 'authentication': { 'rewind': { 'username': self.environ['PATRONI_REWIND_USERNAME'], - 'password': self.no_value_msg} + 'password': NO_VALUE_MSG} }, } } @@ -230,7 +226,7 @@ class TestGenerateConfig(unittest.TestCase): } }, 'postgresql': { - 'connect_address': f'{self._IP}:1984', + 'connect_address': f'{IP}:1984', 'authentication': { 'superuser': { 'username': self.environ['PGUSER'], @@ -330,5 +326,5 @@ class TestGenerateConfig(unittest.TestCase): def test_get_address(self): with patch('socket.getaddrinfo', Mock(side_effect=Exception)), \ patch('logging.warning') as mock_warning: - self.assertEqual(get_address(), (self.no_value_msg, self.no_value_msg)) + self.assertEqual(get_address(), (NO_VALUE_MSG, NO_VALUE_MSG)) self.assertIn('Failed to obtain address: %r', mock_warning.call_args_list[0][0]) From 27915984b488013d82785642fdecb58a52a254c4 Mon Sep 17 00:00:00 2001 From: Polina Bungina <27892524+hughcapet@users.noreply.github.com> Date: Wed, 27 Sep 2023 12:19:58 +0200 Subject: [PATCH 03/11] Add contrib requirement for tests, small docs refactoring (#2887) --- docs/contributing_guidelines.rst | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/docs/contributing_guidelines.rst b/docs/contributing_guidelines.rst index 05d98987..bfa1f4c5 100644 --- a/docs/contributing_guidelines.rst +++ b/docs/contributing_guidelines.rst @@ -3,21 +3,25 @@ Contributing guidelines ======================= -Wanna contribute to Patroni? Yay - here is how! - Chatting -------- -Just want to chat with other Patroni users? Looking for interactive troubleshooting help? Join us on channel `#patroni `__ in the `PostgreSQL Slack `__. +If you have a question, looking for an interactive troubleshooting help or want to chat with other Patroni users, join us on channel `#patroni `__ in the `PostgreSQL Slack `__. + +Reporting bugs +-------------- + +Before reporting a bug please make sure to **reproduce it with the latest Patroni version**! +Also please double check if the issue already exists in our `Issues Tracker `__. Running tests ------------- Requirements for running behave tests: -1. PostgreSQL packages need to be installed. -2. PostgreSQL binaries must be available in your `PATH`. You may need to add them to the path with something like `PATH=/usr/lib/postgresql/11/bin:$PATH python -m behave`. -3. If you'd like to test with external DCSs (e.g., Etcd, Consul, and Zookeeper) you'll need the packages installed and respective services running and accepting unencrypted/unprotected connections on localhost and default port. In the case of Etcd or Consul, the behave test suite could start them up if binaries are available in the `PATH`. +#. PostgreSQL packages including `contrib `__ modules need to be installed. +#. PostgreSQL binaries must be available in your `PATH`. You may need to add them to the path with something like `PATH=/usr/lib/postgresql/11/bin:$PATH python -m behave`. +#. If you'd like to test with external DCSs (e.g., Etcd, Consul, and Zookeeper) you'll need the packages installed and respective services running and accepting unencrypted/unprotected connections on localhost and default port. In the case of Etcd or Consul, the behave test suite could start them up if binaries are available in the `PATH`. Install dependencies: @@ -163,19 +167,12 @@ the watchdog behave feature test scenario with all versions of Postgres. Of course you can combine the two. -Reporting issues ----------------- - -If you have a question about patroni or have a problem using it, please read the :ref:`README ` before filing an issue. -Also double check with the current issues on our `Issues Tracker `__. - Contributing a pull request --------------------------- -1) Submit a comment to the relevant issue or create a new issue describing your proposed change. -2) Do a fork, develop and test your code changes. -3) Include documentation -4) Submit a pull request. +#. Fork the repository, develop and test your code changes. +#. Reflect changes in the user documentation. +#. Submit a pull request with a clear description of the changes objective. Link an existing issue if necessary. You'll get feedback about your pull request as soon as possible. From aaac6f6fb0b4596476e2eef2441c93d78f59393f Mon Sep 17 00:00:00 2001 From: Polina Bungina <27892524+hughcapet@users.noreply.github.com> Date: Wed, 27 Sep 2023 15:57:09 +0200 Subject: [PATCH 04/11] Don't fail if pg_hba/pg_ident contain comment lines (#2888) yaml parser interprets such lines as null and stores it as None into the array of the parsed values, which can not be handled by write() function and crashes the whole bootstrap process. Even though it is not the proper value, it won't hurt if we just ignore it instead of failing completely. --- patroni/postgresql/config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/patroni/postgresql/config.py b/patroni/postgresql/config.py index 40d87f35..b15f7f23 100644 --- a/patroni/postgresql/config.py +++ b/patroni/postgresql/config.py @@ -244,9 +244,10 @@ class ConfigWriter(object): self._fd.write(line) self._fd.write('\n') - def writelines(self, lines: List[str]) -> None: + def writelines(self, lines: List[Optional[str]]) -> None: for line in lines: - self.writeline(line) + if isinstance(line, str): + self.writeline(line) @staticmethod def escape(value: Any) -> str: # Escape (by doubling) any single quotes or backslashes in given string From f77073c8e1ba599bb46adfeb4dba2401e3fd3d9e Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 28 Sep 2023 10:44:11 +0200 Subject: [PATCH 05/11] Speed up dcs failsafe behave tests (#2890) - get rid from sleeps - reduce retry_timeout - avoid graceful Patroni shut down while DCS is "paused", just kill Patroni and after that gracefully stop postgres - don't try to delete Pod when Patroni is killed. If K8s API is paused it takes ages The run time on my laptop is reduced from 2m to 1m28s. --- features/dcs_failsafe_mode.feature | 9 ++++----- features/environment.py | 5 +++-- features/steps/basic_replication.py | 7 ++++++- features/steps/cascading_replication.py | 2 +- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/features/dcs_failsafe_mode.feature b/features/dcs_failsafe_mode.feature index 4a34b754..8345d8ea 100644 --- a/features/dcs_failsafe_mode.feature +++ b/features/dcs_failsafe_mode.feature @@ -4,8 +4,8 @@ Feature: dcs failsafe mode Scenario: check failsafe mode can be successfully enabled Given I start postgres0 And postgres0 is a leader after 10 seconds - And I sleep for 3 seconds - When I issue a PATCH request to http://127.0.0.1:8008/config with {"loop_wait": 2, "ttl": 20, "retry_timeout": 5, "failsafe_mode": true} + Then "config" key in DCS has ttl=30 after 10 seconds + When I issue a PATCH request to http://127.0.0.1:8008/config with {"loop_wait": 2, "ttl": 20, "retry_timeout": 3, "failsafe_mode": true} Then I receive a response code 200 And Response on GET http://127.0.0.1:8008/failsafe contains postgres0 after 10 seconds When I issue a GET request to http://127.0.0.1:8008/failsafe @@ -28,7 +28,6 @@ Feature: dcs failsafe mode When I do a backup of postgres0 And I shut down postgres0 When I start postgres1 in a cluster batman from backup with no_leader - And I sleep for 2 seconds Then postgres1 role is the replica after 12 seconds Scenario: check leader and replica are both in /failsafe key after leader is back @@ -59,12 +58,12 @@ Feature: dcs failsafe mode Given DCS is down And I kill postgres1 And I kill postmaster on postgres1 - And I sleep for 2 seconds Then postgres0 role is the replica after 12 seconds @dcs-failsafe Scenario: check known replica is promoted when leader is down and DCS is up - Given I shut down postgres0 + Given I kill postgres0 + And I shut down postmaster on postgres0 And DCS is up When I start postgres1 Then "members/postgres1" key in DCS has state=running after 10 seconds diff --git a/features/environment.py b/features/environment.py index e3c21252..79db5d16 100644 --- a/features/environment.py +++ b/features/environment.py @@ -162,9 +162,10 @@ class PatroniController(AbstractController): def stop(self, kill=False, timeout=15, postgres=False): if postgres: - return subprocess.call(['pg_ctl', '-D', self._data_dir, 'stop', '-mi', '-w']) + mode = 'i' if kill else 'f' + return subprocess.call(['pg_ctl', '-D', self._data_dir, 'stop', '-m' + mode, '-w']) super(PatroniController, self).stop(kill, timeout) - if isinstance(self._context.dcs_ctl, KubernetesController): + if isinstance(self._context.dcs_ctl, KubernetesController) and not kill: self._context.dcs_ctl.delete_pod(self._name[8:]) if self.watchdog: self.watchdog.stop() diff --git a/features/steps/basic_replication.py b/features/steps/basic_replication.py index 6718cb01..5977eb61 100644 --- a/features/steps/basic_replication.py +++ b/features/steps/basic_replication.py @@ -35,11 +35,16 @@ def kill_patroni(context, name): return context.pctl.stop(name, kill=True) -@step('I kill postmaster on {name:w}') +@step('I shut down postmaster on {name:w}') def stop_postgres(context, name): return context.pctl.stop(name, postgres=True) +@step('I kill postmaster on {name:w}') +def kill_postgres(context, name): + return context.pctl.stop(name, kill=True, postgres=True) + + @step('I add the table {table_name:w} to {pg_name:w}') def add_table(context, table_name, pg_name): # parse the configuration file and get the port diff --git a/features/steps/cascading_replication.py b/features/steps/cascading_replication.py index 9783fbae..c6b43f31 100644 --- a/features/steps/cascading_replication.py +++ b/features/steps/cascading_replication.py @@ -28,7 +28,7 @@ def check_member(context, name, key, value, time_limit): while time.time() < max_time: try: response = json.loads(context.dcs_ctl.query(name)) - dcs_value = response.get(key) + dcs_value = str(response.get(key)) if dcs_value == value: return except Exception: From a329a9d3205871931ec076c0a9eadb54def2021e Mon Sep 17 00:00:00 2001 From: Israel Date: Wed, 4 Oct 2023 06:43:38 -0300 Subject: [PATCH 06/11] Add a documentation page for `patronictl` (#2874) This PR introduces a documentation page for `patronictl` application. We adopted a top-down approach when writing this document. We start by describing the outer most parts, and then keep writing new sections that specialize the knowledge. We basically added a section called `patronictl` to the left menu. Inside that section we created a page with this structure: - `patronictl`: describes what it is - `Configuraiton`: how to configure `patronictl` - `Usage`: how to use the CLI. Inside this section, there are subsections for each of the subcommands exposed by `patronictl`, and each of them are described using the following subsubsections: - `Synopsis`: syntax of the command and its positional and optional arguments - `Description`: a description of what the command does - `Parameters`: a detailed description of the arguments and how to use them - `Examples`: one or more examples of execution of the command References: PAT-200. --- docs/ENVIRONMENT.rst | 6 +- docs/citus.rst | 14 +- docs/dcs_failsafe_mode.rst | 2 +- docs/dynamic_configuration.rst | 2 +- docs/existing_data.rst | 6 +- docs/index.rst | 1 + docs/patroni_configuration.rst | 10 +- docs/patronictl.rst | 1975 ++++++++++++++++++++++++++++++++ docs/pause.rst | 2 +- docs/replica_bootstrap.rst | 2 +- docs/rest_api.rst | 14 +- docs/security.rst | 6 +- docs/yaml_configuration.rst | 10 +- 13 files changed, 2013 insertions(+), 37 deletions(-) create mode 100644 docs/patronictl.rst diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst index e3762aca..f859649d 100644 --- a/docs/ENVIRONMENT.rst +++ b/docs/ENVIRONMENT.rst @@ -209,10 +209,10 @@ REST API CTL --- - **PATRONICTL\_CONFIG\_FILE**: (optional) location of the configuration file. -- **PATRONI\_CTL\_USERNAME**: (optional) Basic-auth username for accessing protected REST API endpoints. If not provided patronictl will use the value provided for REST API "username" parameter. -- **PATRONI\_CTL\_PASSWORD**: (optional) Basic-auth password for accessing protected REST API endpoints. If not provided patronictl will use the value provided for REST API "password" parameter. +- **PATRONI\_CTL\_USERNAME**: (optional) Basic-auth username for accessing protected REST API endpoints. If not provided :ref:`patronictl` will use the value provided for REST API "username" parameter. +- **PATRONI\_CTL\_PASSWORD**: (optional) Basic-auth password for accessing protected REST API endpoints. If not provided :ref:`patronictl` will use the value provided for REST API "password" parameter. - **PATRONI\_CTL\_INSECURE**: (optional) Allow connections to REST API without verifying SSL certs. -- **PATRONI\_CTL\_CACERT**: (optional) Specifies the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs. If not provided patronictl will use the value provided for REST API "cafile" parameter. +- **PATRONI\_CTL\_CACERT**: (optional) Specifies the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs. If not provided :ref:`patronictl` will use the value provided for REST API "cafile" parameter. - **PATRONI\_CTL\_CERTFILE**: (optional) Specifies the file with the client certificate in the PEM format. - **PATRONI\_CTL\_KEYFILE**: (optional) Specifies the file with the client secret key in the PEM format. - **PATRONI\_CTL\_KEYFILE\_PASSWORD**: (optional) Specifies a password for decrypting the client keyfile. diff --git a/docs/citus.rst b/docs/citus.rst index 030cf93e..084931df 100644 --- a/docs/citus.rst +++ b/docs/citus.rst @@ -57,7 +57,7 @@ clusters that are just logically groupped together using the PostgreSQL. Therefore in most cases it is not possible to manage them as a single entity. -It results in two major differences in ``patronictl`` behaviour when +It results in two major differences in :ref:`patronictl` behaviour when ``patroni.yaml`` has the ``citus`` section comparing with the usual: 1. The ``list`` and the ``topology`` by default output all members of the Citus @@ -65,12 +65,12 @@ It results in two major differences in ``patronictl`` behaviour when which Citus group they belong to. 2. For all ``patronictl`` commands the new option is introduced, named ``--group``. For some commands the default value for the group might be - taken from the ``patroni.yaml``. For example, ``patronictl pause`` will + taken from the ``patroni.yaml``. For example, :ref:`patronictl_pause` will enable the maintenance mode by default for the ``group`` that is set in the - ``citus`` section, but for example for ``patronictl switchover`` or - ``patronictl remove`` the group must be explicitly specified. + ``citus`` section, but for example for :ref:`patronictl_switchover` or + :ref:`patronictl_remove` the group must be explicitly specified. -An example of ``patronictl list`` output for the Citus cluster:: +An example of :ref:`patronictl_list` output for the Citus cluster:: postgres@coord1:~$ patronictl list demo + Citus cluster: demo ----------+--------------+---------+----+-----------+ @@ -115,7 +115,7 @@ the coordinator for the shards hosted on a worker node. The switchover then happens while the traffic is kept on the coordinator, and resumes as soon as a new primary worker node is ready to accept read-write queries. -An example of ``patronictl switchover`` on the worker cluster:: +An example of :ref:`patronictl_switchover` on the worker cluster:: postgres@coord1:~$ patronictl switchover demo + Citus cluster: demo ----------+--------------+---------+----+-----------+ @@ -343,7 +343,7 @@ Citus upgrades and PostgreSQL major upgrades First, please read about upgrading Citus version in the `documentation`__. There is one minor change in the process. When executing upgrade, you have to -use ``patronictl restart`` instead of ``systemctl restart`` to restart +use :ref:`patronictl_restart` instead of ``systemctl restart`` to restart PostgreSQL. __ https://docs.citusdata.com/en/latest/admin_guide/upgrading_citus.html diff --git a/docs/dcs_failsafe_mode.rst b/docs/dcs_failsafe_mode.rst index e4eb6061..e6ce363e 100644 --- a/docs/dcs_failsafe_mode.rst +++ b/docs/dcs_failsafe_mode.rst @@ -60,4 +60,4 @@ F.A.Q. - How to enable the Failsafe Mode? - Before enabling the ``failsafe_mode`` please make sure that Patroni version on all members is up-to-date. After that, you can use either the ``PATCH /config`` :ref:`REST API ` or ``patronictl edit-config -s failsafe_mode=true`` + Before enabling the ``failsafe_mode`` please make sure that Patroni version on all members is up-to-date. After that, you can use either the ``PATCH /config`` :ref:`REST API ` or :ref:`patronictl edit-config -s failsafe_mode=true ` diff --git a/docs/dynamic_configuration.rst b/docs/dynamic_configuration.rst index 7f04ce33..285bcab3 100644 --- a/docs/dynamic_configuration.rst +++ b/docs/dynamic_configuration.rst @@ -6,7 +6,7 @@ Dynamic Configuration Settings Dynamic configuration is stored in the DCS (Distributed Configuration Store) and applied on all cluster nodes. -In order to change the dynamic configuration you can use either ``patronictl edit-config`` tool or Patroni :ref:`REST API `. +In order to change the dynamic configuration you can use either :ref:`patronictl_edit_config` tool or Patroni :ref:`REST API `. - **loop\_wait**: the number of seconds the loop will sleep. Default value: 10 - **ttl**: the TTL to acquire the leader lock (in seconds). Think of it as the length of time before initiation of the automatic failover process. Default value: 30 diff --git a/docs/existing_data.rst b/docs/existing_data.rst index cb07bfa9..442b0b1c 100644 --- a/docs/existing_data.rst +++ b/docs/existing_data.rst @@ -42,12 +42,12 @@ You can find below an overview of steps for converting an existing Postgres clus #. Start Patroni using the ``patroni`` systemd service unit. It automatically detects that Postgres is already running and starts monitoring the instance. -#. Hand over Postgres "start up procedure" to Patroni. In order to do that you need to restart the cluster members through ``patronictl restart cluster-name member-name`` command. For minimal downtime you might want to split this step into: +#. Hand over Postgres "start up procedure" to Patroni. In order to do that you need to restart the cluster members through :ref:`patronictl restart cluster-name member-name ` command. For minimal downtime you might want to split this step into: #. Immediate restart of the standby nodes. #. Scheduled restart of the primary node within a maintenance window. -#. If you configured permanent slots in step ``1.2.``, then you should remove them from ``slots`` configuration through ``patronictl edit-config cluster-name member-name`` command once the ``restart_lsn`` of the slots created by Patroni is able to catch up with the ``restart_lsn`` of the original slots for the corresponding members. By removing the slots from ``slots`` configuration you will allow Patroni to drop the original slots from your cluster once they are not needed anymore. You can find below an example query to check the ``restart_lsn`` of a couple slots, so you can compare them: +#. If you configured permanent slots in step ``1.2.``, then you should remove them from ``slots`` configuration through :ref:`patronictl edit-config cluster-name member-name ` command once the ``restart_lsn`` of the slots created by Patroni is able to catch up with the ``restart_lsn`` of the original slots for the corresponding members. By removing the slots from ``slots`` configuration you will allow Patroni to drop the original slots from your cluster once they are not needed anymore. You can find below an example query to check the ``restart_lsn`` of a couple slots, so you can compare them: .. code-block:: sql @@ -73,7 +73,7 @@ The only possible way to do a major upgrade currently is: #. Stop Patroni #. Upgrade PostgreSQL binaries and perform `pg_upgrade `_ on the primary node #. Update patroni.yml -#. Remove the initialize key from DCS or wipe complete cluster state from DCS. The second one could be achieved by running ``patronictl remove ``. It is necessary because pg_upgrade runs initdb which actually creates a new database with a new PostgreSQL system identifier. +#. Remove the initialize key from DCS or wipe complete cluster state from DCS. The second one could be achieved by running :ref:`patronictl remove cluster-name ` . It is necessary because pg_upgrade runs initdb which actually creates a new database with a new PostgreSQL system identifier. #. If you wiped the cluster state in the previous step, you may wish to copy patroni.dynamic.json from old data dir to the new one. It will help you to retain some PostgreSQL parameters you had set before. #. Start Patroni on the primary node. #. Upgrade PostgreSQL binaries, update patroni.yml and wipe the data_dir on standby nodes. diff --git a/docs/index.rst b/docs/index.rst index c8f94aaf..f5a4f2d4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,6 +25,7 @@ Currently supported PostgreSQL versions: 9.3 to 16. installation patroni_configuration rest_api + patronictl replica_bootstrap replication_modes watchdog diff --git a/docs/patroni_configuration.rst b/docs/patroni_configuration.rst index ef5ca020..3978e48d 100644 --- a/docs/patroni_configuration.rst +++ b/docs/patroni_configuration.rst @@ -15,7 +15,7 @@ There are 3 types of Patroni configuration: - Global :ref:`dynamic configuration `. These options are stored in the DCS (Distributed Configuration Store) and applied on all cluster nodes. - Dynamic configuration can be set at any time using ``patronictl edit-config`` tool or Patroni :ref:`REST API `. + Dynamic configuration can be set at any time using :ref:`patronictl_edit_config` tool or Patroni :ref:`REST API `. If the options changed are not part of the startup configuration, they are applied asynchronously (upon the next wake up cycle) to every node, which gets subsequently reloaded. If the node requires a restart to apply the configuration (for `PostgreSQL parameters `__ with context postmaster, if their values @@ -24,7 +24,7 @@ There are 3 types of Patroni configuration: - Local :ref:`configuration file ` (patroni.yml). These options are defined in the configuration file and take precedence over dynamic configuration. - ``patroni.yml`` can be changed and reloaded at runtime (without restart of Patroni) by sending SIGHUP to the Patroni process, performing ``POST /reload`` REST-API request or executing ``patronictl reload``. Local configuration can be either a single YAML file or a directory. When it is a directory, all YAML files in that directory are loaded one by one in sorted order. In case a key is defined in multiple files, the occurrence in the last file takes precedence. + ``patroni.yml`` can be changed and reloaded at runtime (without restart of Patroni) by sending SIGHUP to the Patroni process, performing ``POST /reload`` REST-API request or executing :ref:`patronictl_reload`. Local configuration can be either a single YAML file or a directory. When it is a directory, all YAML files in that directory are loaded one by one in sorted order. In case a key is defined in multiple files, the occurrence in the last file takes precedence. - :ref:`Environment configuration `. It is possible to set/override some of the "Local" configuration parameters with environment variables. @@ -105,10 +105,10 @@ Changing these parameters require a PostgreSQL restart to take effect, and their As explained before, Patroni restrict changing their values through :ref:`dynamic configuration `, which usually consists of: -1. Applying changes through ``patronictl edit-config`` (or via REST API ``/config`` endpoint) -2. Restarting nodes through ``patronictl restart`` (or via REST API ``/restart`` endpoint) +1. Applying changes through :ref:`patronictl_edit_config` (or via REST API ``/config`` endpoint) +2. Restarting nodes through :ref:`patronictl_restart` (or via REST API ``/restart`` endpoint) -**Note:** please keep in mind that you should perform a restart of the PostgreSQL nodes through ``patronictl restart`` command, or via REST API ``/restart`` endpoint. An attempt to restart PostgreSQL by restarting the Patroni daemon, e.g. by executing ``systemctl restart patroni``, can cause a failover to occur in the cluster, if you are restarting the primary node. +**Note:** please keep in mind that you should perform a restart of the PostgreSQL nodes through :ref:`patronictl_restart` command, or via REST API ``/restart`` endpoint. An attempt to restart PostgreSQL by restarting the Patroni daemon, e.g. by executing ``systemctl restart patroni``, can cause a failover to occur in the cluster, if you are restarting the primary node. However, as those settings manage shared memory, some extra care should be taken when restarting the nodes: diff --git a/docs/patronictl.rst b/docs/patronictl.rst new file mode 100644 index 00000000..8b8cfb7e --- /dev/null +++ b/docs/patronictl.rst @@ -0,0 +1,1975 @@ +.. _patronictl: + +patronictl +========== + +Patroni has a command-line interface named ``patronictl``, which is used basically to interact with Patroni's REST API and with the DCS. It is intended to make it easier to perform operations in the cluster, and can easily be used by humans or scripts. + +.. _patronictl_configuration: + +Configuration +------------- + +``patronictl`` uses 3 sections of the configuration: + +- **ctl**: how to authenticate against the Patroni REST API, and how to validate the server identity. Refer to :ref:`ctl settings ` for more details; +- **restapi**: how to authenticate against the Patroni REST API, and how to validate the server identity. Only used if ``ctl`` configuration is not enough. ``patronictl`` is mainly interested in ``restapi.authentication`` section (in case ``ctl.authentication`` is missing) and ``restapi.cafile`` setting (in case ``ctl.cacert`` is missing). Refer to :ref:`REST API settings ` for more details; +- DCS (e.g. **etcd**): how to contact and authenticate against the DCS used by Patroni. + +Those configuration options can come either from environment variables or from a configuration file. Look for the above sections in :ref:`Environment Configuration Settings ` or :ref:`YAML Configuration Settings ` to understand how you can set the options for them through environment variables or through a configuration file. + +If you opt for using environment variables, it's a straight forward approach. Patronictl will read the environment variables and use their values. + +If you opt for using a configuration file, you have different ways to inform ``patronictl`` about the file to be used. By default ``patronictl`` will attempt to load a configuration file named ``patronictl.yaml``, which is expected to be found under either of these paths, according to your system: + +- Mac OS X: ``~/Library/Application Support/patroni`` +- Mac OS X (POSIX): ``~/.patroni`` +- Unix: ``~/.config/patroni`` +- Unix (POSIX): ``~/.patroni`` +- Windows (roaming): ``C:\Users\\AppData\Roaming\patroni`` +- Windows (not roaming): ``C:\Users\\AppData\Local\patroni`` + +You can override that behavior either by: + +- Setting the environment variable ``PATRONICTL_CONFIG_FILE`` with the path to a custom configuration file; +- Using the ``-c`` / ``--config-file`` command-line argument of ``patronictl`` with the path to a custom configuration file. + +.. note:: + If you are running ``patronictl`` in the same host as ``patroni`` daemon is running, you may just use the same configuration file if it contains all the configuration sections required by ``patronictl``. + +.. _patronictl_usage: + +Usage +----- + +``patronictl`` exposes several handy operations. This section is intended to describe each of them. + +Before jumping into each of the sub-commands of ``patronictl``, be aware that ``patronictl`` itself has the following command-line arguments: + +``-c`` / ``--config-file`` + As explained before, used to provide a path to a configuration file for ``patronictl``. + +``-d`` / ``--dcs-url`` / ``--dcs`` + Provide a connection string to the DCS used by Patroni. + + This argument can be used either to override the DCS settings from the ``patronictl`` configuration, or to define it if it's missing in the configuration. + + The value should be in the format ``DCS://HOST:PORT``, e.g. ``etcd3://localhost:2379`` to connect to etcd v3 running on ``localhost``. + +``-k`` / ``--insecure`` + Flag to bypass validation of REST API server SSL certificate. + +This is the synopsis for running a command from the ``patronictl``: + +.. code:: text + + patronictl [ { -c | --config-file } CONFIG_FILE ] + [ { -d | --dcs-url | --dcs } DCS_URL ] + [ { -k | --insecure } ] + SUBCOMMAND + +.. note:: + + This is the syntax for the synopsis: + + - Options between square brackets are optional; + - Options between curly brackets represent a "choose one of set" operation; + - Options with ``[, ... ]`` can be specified multiple times; + - Things written in uppercase represent a literal that should be given a value to. + + We will use this same syntax when describing ``patronictl`` sub-commands in the following sub-sections. + Also, when describing sub-commands in the following sub-sections, the commands' synposis should be seen as a replacement for the ``SUBCOMMAND`` in the above synopsis. + +In the following sub-sections you can find a description of each command implemented by ``patronictl``. For sake of example, we will use the configuration files present in the GitHub repository of Patroni (files ``postgres0.yml``, ``postgres1.yml`` and ``postgres2.yml``). + +.. _patronictl_dsn: + +patronictl dsn +^^^^^^^^^^^^^^ + +.. _patronictl_dsn_synopsis: + +Synopsis +"""""""" + +.. code:: text + + dsn + [ CLUSTER_NAME ] + [ { { -r | --role } { leader | primary | standby-leader | replica | standby | any } | { -m | --member } MEMBER_NAME } ] + [ --group CITUS_GROUP ] + +.. _patronictl_dsn_description: + +Description +""""""""""" + +``patronictl dsn`` gets the connection string for one member of the Patroni cluster. + +If multiple members match the parameters of this command, one of them will be chosen, prioritizing the primary node. + +.. _patronictl_dsn_parameters: + +Parameters +"""""""""" + +``CLUSTER_NAME`` + Name of the Patroni cluster. + + If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. + +``-r`` / ``--role`` + Choose a member that has the given role. + + Role can be one of: + + - ``leader``: the leader of either a regular Patroni cluster or a standby Patroni cluster; or + - ``primary``: the leader of a regular Patroni cluster; or + - ``standby-leader``: the leader of a standby Patroni cluster; or + - ``replica``: a replica of a Patroni cluster; or + - ``standby``: same as ``replica``; or + - ``any``: any role. Same as omitting this parameter; or + +``-m`` / ``--member`` + Choose a member of the cluster with the given name. + + ``MEMBER_NAME`` is the name of the member. + +``--group`` + Choose a member that is part of the given Citus group. + + ``CITUS_GROUP`` is the ID of the Citus group. + +.. _patronictl_dsn_examples: + +Examples +"""""""" + +Get DSN of the primary node: + +.. code:: bash + + $ patronictl -c postgres0.yml dsn batman -r primary + host=127.0.0.1 port=5432 + +Get DSN of the node named ``postgresql1``: + +.. code:: bash + + $ patronictl -c postgres0.yml dsn batman --member postgresql1 + host=127.0.0.1 port=5433 + +.. _patronictl_edit_config: + +patronictl edit-config +^^^^^^^^^^^^^^^^^^^^^^ + +.. _patronictl_edit_config_synopsis: + +Synopsis +"""""""" + +.. code:: text + + edit-config + [ CLUSTER_NAME ] + [ --group CITUS_GROUP ] + [ { -q | --quiet } ] + [ { -s | --set } CONFIG="VALUE" [, ... ] ] + [ { -p | --pg } PG_CONFIG="PG_VALUE" [, ... ] ] + [ { --apply | --replace } CONFIG_FILE ] + [ --force ] + +.. _patronictl_edit_config_description: + +Description +""""""""""" + +``patronictl edit-config`` changes the dynamic configuration of the cluster and updates the DCS with that. + +.. note:: + When invoked through a TTY the command attempts to show a diff of the dynamic configuration through a pager. By default, it attempts to use either ``less`` or ``more``. If you want a different pager, set the ``PAGER`` environment variable with the desired one. + +.. _patronictl_edit_config_parameters: + +Parameters +"""""""""" + +``CLUSTER_NAME`` + Name of the Patroni cluster. + + If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. + +``--group`` + Change dynamic configuration of the given Citus group. + + If not given, ``patronictl`` will attempt to fetch that from the ``citus.group`` configuration, if it exists. + + ``CITUS_GROUP`` is the ID of the Citus group. + +``-q`` / ``--quiet`` + Flag to skip showing the configuration diff. + +``-s`` / ``--set`` + Set a given dynamic configuration option with a given value. + + ``CONFIG`` is the name of the dynamic configuration path in the YAML tree, with levels joined by ``.`` . + + ``VALUE`` is the value for ``CONFIG``. If it is ``null``, then ``CONFIG`` will be removed from the dynamic configuration. + +``-p`` / ``--pg`` + Set a given dynamic Postgres configuration option with the given value. + + It is essentially a shorthand for ``--s`` / ``--set`` with ``CONFIG`` prepended with ``postgresql.parameters.``. + + ``PG_CONFIG`` is the name of the Postgres configuration to be set. + + ``PG_VALUE`` is the value for ``PG_CONFIG``. If it is ``nulll``, then ``PG_CONFIG`` will be removed from the dynamic configuration. + +``--apply`` + Apply dynamic configuration from the given file. + + It is similar to specifying multiple ``-s`` / ``--set`` options, one for each configuration from ``CONFIG_FILE``. + + ``CONFIG_FILE`` is the path to a file containing the dynamic configuration to be applied, in YAML format. Use ``-`` if you want to read from ``stdin``. + +``--replace`` + Replace the dynamic configuration in the DCS with the dynamic configuration specified in the given file. + + ``CONFIG_FILE`` is the path to a file containing the new dynamic configuration to take effect, in YAML format. Use ``-`` if you want to read from ``stdin``. + +``--force`` + Flag to skip confirmation prompts when changing the dynamic configuration. + + Useful for scripts. + +.. _patronictl_edit_config_examples: + +Examples +"""""""" + +Change ``max_connections`` Postgres GUC: + +.. code:: diff + + patronictl -c postgres0.yml edit-config batman --pg max_connections="150" --force + --- + +++ + @@ -1,6 +1,8 @@ + loop_wait: 10 + maximum_lag_on_failover: 1048576 + postgresql: + + parameters: + + max_connections: 150 + pg_hba: + - host replication replicator 127.0.0.1/32 md5 + - host all all 0.0.0.0/0 md5 + + Configuration changed + +Change ``loop_wait`` and ``ttl`` settings: + +.. code:: diff + + patronictl -c postgres0.yml edit-config batman --set loop_wait="15" --set ttl="45" --force + --- + +++ + @@ -1,4 +1,4 @@ + -loop_wait: 10 + +loop_wait: 15 + maximum_lag_on_failover: 1048576 + postgresql: + pg_hba: + @@ -6,4 +6,4 @@ + - host all all 0.0.0.0/0 md5 + use_pg_rewind: true + retry_timeout: 10 + -ttl: 30 + +ttl: 45 + + Configuration changed + +Remove ``maximum_lag_on_failover`` setting from dynamic configuration: + +.. code:: diff + + patronictl -c postgres0.yml edit-config batman --set maximum_lag_on_failover="null" --force + --- + +++ + @@ -1,5 +1,4 @@ + loop_wait: 10 + -maximum_lag_on_failover: 1048576 + postgresql: + pg_hba: + - host replication replicator 127.0.0.1/32 md5 + + Configuration changed + +.. _patronictl_failover: + +patronictl failover +^^^^^^^^^^^^^^^^^^^ + +.. _patronictl_failover_synopsis: + +Synopsis +"""""""" + +.. code:: text + + failover + [ CLUSTER_NAME ] + [ --group CITUS_GROUP ] + [ { --leader | --primary } LEADER_NAME ] + --candidate CANDIDATE_NAME + [ --force ] + +.. _patronictl_failover_description: + +Description +""""""""""" + +``patronictl failover`` performs a manual failover in the cluster. + +It is designed to be used when the cluster is not healthy, e.g.: + +- There is no leader; or +- There is no synchronous standby available in a synchronous cluster. + +It also allows to fail over to an asynchronous node if synchronous mode is enabled. + +.. note:: + Nothing prevents you from running ``patronictl failover`` in a healthy cluster. However, we recommend using ``patronictl switchover`` in those cases. + +.. warning:: + Triggering a failover can cause data loss depending on how up-to-date the promoted replica is in comparison to the primary. + +.. _patronictl_failover_parameters: + +Parameters +"""""""""" + +``CLUSTER_NAME`` + Name of the Patroni cluster. + + If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. + +``--group`` + Perform a failover in the given Citus group. + + ``CITUS_GROUP`` is the ID of the Citus group. + +``--leader`` / ``--primary`` + Indicate who is the expected leader at failover time. + + If given, a switchover is performed instead of a failover. + + ``LEADER_NAME`` should match the name of the current leader in the cluster. + + .. warning:: + This argument is deprecated and will be removed in a future release. + +``--candidate`` + The node to be promoted on failover. + + ``CANDIDATE_NAME`` is the name of the node to be promoted. + +``--force`` + Flag to skip confirmation prompts when performing the failover. + + Useful for scripts. + +.. _patronictl_failover_examples: + +Examples +"""""""" + +Fail over to node ``postgresql2``: + +.. code:: bash + + $ patronictl -c postgres0.yml failover batman --candidate postgresql2 --force + Current cluster topology + + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +-------------+----------------+---------+-----------+----+-----------+ + | postgresql0 | 127.0.0.1:5432 | Leader | running | 3 | | + | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 3 | 0 | + | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 3 | 0 | + +-------------+----------------+---------+-----------+----+-----------+ + 2023-09-12 11:52:27.50978 Successfully failed over to "postgresql2" + + Cluster: batman (7277694203142172922) -+---------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +-------------+----------------+---------+---------+----+-----------+ + | postgresql0 | 127.0.0.1:5432 | Replica | stopped | | unknown | + | postgresql1 | 127.0.0.1:5433 | Replica | running | 3 | 0 | + | postgresql2 | 127.0.0.1:5434 | Leader | running | 3 | | + +-------------+----------------+---------+---------+----+-----------+ + +.. _patronictl_flush: + +patronictl flush +^^^^^^^^^^^^^^^^ + +.. _patronictl_flush_synopsis: + +Synopsis +"""""""" + +.. code:: text + + flush + CLUSTER_NAME + [ MEMBER_NAME [, ... ] ] + { restart | switchover } + [ --group CITUS_GROUP ] + [ { -r | --role } { leader | primary | standby-leader | replica | standby | any } ] + [ --force ] + +.. _patronictl_flush_description: + +Description +""""""""""" + +``patronictl flush`` discards scheduled events, if any. + +.. _patronictl_flush_parameters: + +Parameters +"""""""""" + +``CLUSTER_NAME`` + Name of the Patroni cluster. + +``MEMBER_NAME`` + Discard scheduled events for the given Patroni member(s). + + Multiple members can be specified. If no members are specified, all of them are considered. + + .. note:: + Only used if discarding scheduled restart events. + +``restart`` + Discard scheduled restart events. + +``switchover`` + Discard scheduled switchover event. + +``--group`` + Discard scheduled events from the given Citus group. + + ``CITUS_GROUP`` is the ID of the Citus group. + +``-r`` / ``--role`` + Discard scheduled events for members that have the given role. + + Role can be one of: + + - ``leader``: the leader of either a regular Patroni cluster or a standby Patroni cluster; or + - ``primary``: the leader of a regular Patroni cluster; or + - ``standby-leader``: the leader of a standby Patroni cluster; or + - ``replica``: a replica of a Patroni cluster; or + - ``standby``: same as ``replica``; or + - ``any``: any role. Same as omitting this parameter. + + .. note:: + Only used if discarding scheduled restart events. + +``--force`` + Flag to skip confirmation prompts when performing the flush. + + Useful for scripts. + +.. _patronictl_flush_examples: + +Examples +"""""""" + +Discard a scheduled switchover event: + +.. code:: bash + + $ patronictl -c postgres0.yml flush batman switchover --force + Success: scheduled switchover deleted + +Discard scheduled restart of all standby nodes: + +.. code:: bash + + $ patronictl -c postgres0.yml flush batman restart -r replica --force + + Cluster: batman (7277694203142172922) -+-----------+----+-----------+---------------------------+ + | Member | Host | Role | State | TL | Lag in MB | Scheduled restart | + +-------------+----------------+---------+-----------+----+-----------+---------------------------+ + | postgresql0 | 127.0.0.1:5432 | Leader | running | 5 | | 2023-09-12T17:17:00+00:00 | + | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 5 | 0 | 2023-09-12T17:17:00+00:00 | + | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 5 | 0 | 2023-09-12T17:17:00+00:00 | + +-------------+----------------+---------+-----------+----+-----------+---------------------------+ + Success: flush scheduled restart for member postgresql1 + Success: flush scheduled restart for member postgresql2 + +Discard scheduled restart of nodes ``postgresql0`` and ``postgresql1``: + +.. code:: bash + + $ patronictl -c postgres0.yml flush batman postgresql0 postgresql1 restart --force + + Cluster: batman (7277694203142172922) -+-----------+----+-----------+---------------------------+ + | Member | Host | Role | State | TL | Lag in MB | Scheduled restart | + +-------------+----------------+---------+-----------+----+-----------+---------------------------+ + | postgresql0 | 127.0.0.1:5432 | Leader | running | 5 | | 2023-09-12T17:17:00+00:00 | + | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 5 | 0 | 2023-09-12T17:17:00+00:00 | + | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 5 | 0 | 2023-09-12T17:17:00+00:00 | + +-------------+----------------+---------+-----------+----+-----------+---------------------------+ + Success: flush scheduled restart for member postgresql0 + Success: flush scheduled restart for member postgresql1 + +.. _patronictl_history: + +patronictl history +^^^^^^^^^^^^^^^^^^ + +.. _patronictl_history_synopsis: + +Synopsis +"""""""" + +.. code:: text + + history + [ CLUSTER_NAME ] + [ --group CITUS_GROUP ] + [ { -f | --format } { pretty | tsv | json | yaml } ] + +.. _patronictl_history_description: + +Description +""""""""""" + +``patronictl history`` shows a history of failover and switchover events from the cluster, if any. + +The following information is included in the output: + +``TL`` + Postgres timeline at which the event occurred. + +``LSN`` + Postgres LSN at which the event occurred. + +``Reason`` + Reason fetched from the Postgres ``.history`` file. + +``Timestamp`` + Time when the event occurred. + +``New Leader`` + Patroni member that has been promoted during the event. + +.. _patronictl_history_parameters: + +Parameters +"""""""""" + +``CLUSTER_NAME`` + Name of the Patroni cluster. + + If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. + +``--group`` + Show history of events from the given Citus group. + + ``CITUS_GROUP`` is the ID of the Citus group. + + If not given, ``patronictl`` will attempt to fetch that from the ``citus.group`` configuration, if it exists. + +``-f`` / ``--format`` + How to format the list of events in the output. + + Format can be one of: + + - ``pretty``: prints history as a pretty table; or + - ``tsv``: prints history as tabular information, with columns delimited by ``\t``; or + - ``json``: prints history in JSON format; or + - ``yaml``: prints history in YAML format. + + The default is ``pretty``. + +``--force`` + Flag to skip confirmation prompts when performing the flush. + + Useful for scripts. + +.. _patronictl_history_examples: + +Examples +"""""""" + +Show the history of events: + +.. code:: bash + + $ patronictl -c postgres0.yml history batman + +----+----------+------------------------------+----------------------------------+-------------+ + | TL | LSN | Reason | Timestamp | New Leader | + +----+----------+------------------------------+----------------------------------+-------------+ + | 1 | 24392648 | no recovery target specified | 2023-09-11T22:11:27.125527+00:00 | postgresql0 | + | 2 | 50331864 | no recovery target specified | 2023-09-12T11:34:03.148097+00:00 | postgresql0 | + | 3 | 83886704 | no recovery target specified | 2023-09-12T11:52:26.948134+00:00 | postgresql2 | + | 4 | 83887280 | no recovery target specified | 2023-09-12T11:53:09.620136+00:00 | postgresql0 | + +----+----------+------------------------------+----------------------------------+-------------+ + +Show the history of events in YAML format: + +.. code:: bash + + $ patronictl -c postgres0.yml history batman -f yaml + - LSN: 24392648 + New Leader: postgresql0 + Reason: no recovery target specified + TL: 1 + Timestamp: '2023-09-11T22:11:27.125527+00:00' + - LSN: 50331864 + New Leader: postgresql0 + Reason: no recovery target specified + TL: 2 + Timestamp: '2023-09-12T11:34:03.148097+00:00' + - LSN: 83886704 + New Leader: postgresql2 + Reason: no recovery target specified + TL: 3 + Timestamp: '2023-09-12T11:52:26.948134+00:00' + - LSN: 83887280 + New Leader: postgresql0 + Reason: no recovery target specified + TL: 4 + Timestamp: '2023-09-12T11:53:09.620136+00:00' + +.. _patronictl_list: + +patronictl list +^^^^^^^^^^^^^^^ + +.. _patronictl_list_synopsis: + +Synopsis +"""""""" + +.. code:: text + + list + [ CLUSTER_NAME [, ... ] ] + [ --group CITUS_GROUP ] + [ { -e | --extended } ] + [ { -t | --timestamp } ] + [ { -f | --format } { pretty | tsv | json | yaml } ] + [ { -W | { -w | --watch } TIME } ] + +.. _patronictl_list_description: + +Description +""""""""""" + +``patronictl list`` shows information about Patroni cluster and its members. + +The following information is included in the output: + +``Cluster`` + Name of the Patroni cluster. + +``Member`` + Name of the Patroni member. + +``Host`` + Host where the member is located. + +``Role`` + Current role of the member. + + Can be one among: + + * ``Leader``: the current leader of a regular Patroni cluster; or + * ``Standby Leader``: the current leader of a Patroni standby cluster; or + * ``Sync Standby``: a synchronous standby of a Patroni cluster with synchronous mode enabled; or + * ``Replica``: a regular standby of a Patroni cluster. + +``State`` + Current state of Postgres in the Patroni member. + + Some examples among the possible states: + + * ``running``: if Postgres is currently up and running; + * ``streaming``: if a replica and Postgres is currently streaming WALs from the primary node; + * ``in archive recovery``: if a replica and Postgres is currently fetching WALs from the archive; + * ``stopped``: if Postgres had been shut down; + * ``crashed``: if Postgres has crashed. + +``TL`` + Current Postgres timeline in the Patroni member. + +``Lag in MB`` + Amount worth of replication lag in megabytes between the Patroni member and its upstream. + +Besides that, the following information may be included in the output: + +``System identifier`` + Postgres system identifier. + + .. note:: + Shown in the table header. + + Only shown if output format is ``pretty``. + +``Group`` + Citus group ID. + + .. note:: + Shown in the table header. + + Only shown if a Citus cluster. + +``Pending restart`` + ``*`` indicates that the node needs a restart for some Postgres configuration to take effect. An empty value indicates the node does not require a restart. + + .. note:: + Shown as a member attribute. + + Shown if: + + - Printing in ``pretty`` or ``tsv`` format and with extended output enabled; or + - If node requires a restart. + +``Scheduled restart`` + Timestamp at which a restart has been scheduled for the Postgres instance managed by the Patroni member. An empty value indicates there is no scheduled restart for the member. + + .. note:: + Shown as a member attribute. + + Shown if: + + - Printing in ``pretty`` or ``tsv`` format and with extended output enabled; or + - If node has a scheduled restart. + +``Tags`` + Contains tags set for the Patroni member. An empty value indicates that either no tags have been configured, or that they have been configured with default values. + + .. note:: + Shown as a member attribute. + + Shown if: + + - Printing in ``pretty`` or ``tsv`` format and with extended output enabled; or + - If node has any custom tags, or any default tags with non-default values. + +``Scheduled switchover`` + Timestamp at which a switchover has been scheduled for the Patroni cluster, if any. + + .. note:: + Shown in the table footer. + + Only shown if there is a scheduled switchover, and output format is ``pretty``. + +``Maintenance mode`` + + If the cluster monitoring is currently paused. + + .. note:: + Shown in the table footer. + + Only shown if the cluster is paused, and output format is ``pretty``. + +.. _patronictl_list_parameters: + +Parameters +"""""""""" + +``CLUSTER_NAME`` + Name of the Patroni cluster. + + If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. + +``--group`` + Show information about members from the given Citus group. + + ``CITUS_GROUP`` is the ID of the Citus group. + +``-e`` / ``--extended`` + Show extended information. + + Force showing ``Pending restart``, ``Scheduled restart`` and ``Tags`` attributes, even if their value is empty. + + .. note:: + Only applies to ``pretty`` and ``tsv`` output formats. + +``-t`` / ``--timestamp`` + Print timestamp before printing information about the cluster and its members. + +``-f`` / ``--format`` + How to format the list of events in the output. + + Format can be one of: + + - ``pretty``: prints history as a pretty table; or + - ``tsv``: prints history as tabular information, with columns delimited by ``\t``; or + - ``json``: prints history in JSON format; or + - ``yaml``: prints history in YAML format. + + The default is ``pretty``. + +``-W`` + Automatically refresh information every 2 seconds. + +``-w`` / ``--watch`` + Automatically refresh information at the specified interval. + + ``TIME`` is the interval between refreshes, in seconds. + +.. _patronictl_list_examples: + +Examples +"""""""" + +Show information about the cluster in pretty format: + +.. code:: bash + + $ patronictl -c postgres0.yml list batman + + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +-------------+----------------+---------+-----------+----+-----------+ + | postgresql0 | 127.0.0.1:5432 | Leader | running | 5 | | + | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 5 | 0 | + | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 5 | 0 | + +-------------+----------------+---------+-----------+----+-----------+ + +Show information about the cluster in pretty format with extended columns: + +.. code:: bash + + $ patronictl -c postgres0.yml list batman -e + + Cluster: batman (7277694203142172922) -+-----------+----+-----------+-----------------+-------------------+------+ + | Member | Host | Role | State | TL | Lag in MB | Pending restart | Scheduled restart | Tags | + +-------------+----------------+---------+-----------+----+-----------+-----------------+-------------------+------+ + | postgresql0 | 127.0.0.1:5432 | Leader | running | 5 | | | | | + | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 5 | 0 | | | | + | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 5 | 0 | | | | + +-------------+----------------+---------+-----------+----+-----------+-----------------+-------------------+------+ + +Show information about the cluster in YAML format, with timestamp of execution: + +.. code:: bash + + $ patronictl -c postgres0.yml list batman -f yaml -t + 2023-09-12 13:30:48 + - Cluster: batman + Host: 127.0.0.1:5432 + Member: postgresql0 + Role: Leader + State: running + TL: 5 + - Cluster: batman + Host: 127.0.0.1:5433 + Lag in MB: 0 + Member: postgresql1 + Role: Replica + State: streaming + TL: 5 + - Cluster: batman + Host: 127.0.0.1:5434 + Lag in MB: 0 + Member: postgresql2 + Role: Replica + State: streaming + TL: 5 + +.. _patronictl_pause: + +patronictl pause +^^^^^^^^^^^^^^^^ + +.. _patronictl_pause_synopsis: + +Synopsis +"""""""" + +.. code:: text + + pause + [ CLUSTER_NAME ] + [ --group CITUS_GROUP ] + [ --wait ] + +.. _patronictl_pause_description: + +Description +""""""""""" + +``patronictl pause`` temporarily puts the Patroni cluster in maintenance mode and disables automatic failover. + +.. _patronictl_pause_parameters: + +Parameters +"""""""""" + +``CLUSTER_NAME`` + Name of the Patroni cluster. + + If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. + +``--group`` + Pause the given Citus group. + + ``CITUS_GROUP`` is the ID of the Citus group. + + If not given, ``patronictl`` will attempt to fetch that from the ``citus.group`` configuration, if it exists. + +``--wait`` + Wait until all Patroni members are paused before returning control to the caller. + +.. _patronictl_pause_examples: + +Examples +"""""""" + +Put the cluster in maintenance mode, and wait until all nodes have been paused: + +.. code:: bash + + $ patronictl -c postgres0.yml pause batman --wait + 'pause' request sent, waiting until it is recognized by all nodes + Success: cluster management is paused + +.. _patronictl_query: + +patronictl query +^^^^^^^^^^^^^^^^ + +.. _patronictl_query_synopsis: + +Synopsis +"""""""" + +.. code:: text + + query + [ CLUSTER_NAME ] + [ --group CITUS_GROUP ] + [ { { -r | --role } { leader | primary | standby-leader | replica | standby | any } | { -m | --member } MEMBER_NAME } ] + [ { -d | --dbname } DBNAME ] + [ { -U | --username } USERNAME ] + [ --password ] + [ --format { pretty | tsv | json | yaml } ] + [ { { -f | --file } FILE_NAME | { -c | --command } SQL_COMMAND } ] + [ --delimiter ] + [ { -W | { -w | --watch } TIME } ] + +.. _patronictl_query_description: + +Description +""""""""""" + +``patronictl query`` executes a SQL command or script against a member of the Patroni cluster. + +.. _patronictl_query_parameters: + +Parameters +"""""""""" + +``CLUSTER_NAME`` + Name of the Patroni cluster. + + If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. + +``--group`` + Query the given Citus group. + + ``CITUS_GROUP`` is the ID of the Citus group. + +``-r`` / ``--role`` + Choose a member that has the given role. + + Role can be one of: + + - ``leader``: the leader of either a regular Patroni cluster or a standby Patroni cluster; or + - ``primary``: the leader of a regular Patroni cluster; or + - ``standby-leader``: the leader of a standby Patroni cluster; or + - ``replica``: a replica of a Patroni cluster; or + - ``standby``: same as ``replica``; or + - ``any``: any role. Same as omitting this parameter. + +``-m`` / ``--member`` + Choose a member that has the given name. + + ``MEMBER_NAME`` is the name of the member to be picked. + +``-d`` / ``--dbname`` + Database to connect and run the query. + + ``DBNAME`` is the name of the database. If not given, defaults to ``USERNAME``. + +``-U`` / ``--username`` + User to connect to the database. + + ``USERNAME`` name of the user. If not given, defaults to the operating system user running ``patronictl query``. + +``--password`` + Prompt for the password of the connecting user. + + As Patroni uses ``libpq``, alternatively you can create a ``~/.pgpass`` file or set the ``PGPASSWORD`` environment variable. + +``--format`` + How to format the output of the query. + + Format can be one of: + + - ``pretty``: prints query output as a pretty table; or + - ``tsv``: prints query output as tabular information, with columns delimited by ``\t``; or + - ``json``: prints query output in JSON format; or + - ``yaml``: prints query output in YAML format. + + The default is ``tsv``. + +``-f`` / ``--file`` + Use a file as source of commands to run queries. + + ``FILE_NAME`` is the path to the source file. + +``-c`` / ``--command`` + Run the given SQL command in the query. + + ``SQL_COMMAND`` is the SQL command to be executed. + +``--delimiter`` + The delimiter when printing information in ``tsv`` format, or ``\t`` if omitted. + +``-W`` + Automatically re-run the query every 2 seconds. + +``-w`` / ``--watch`` + Automatically re-run the query at the specified interval. + + ``TIME`` is the interval between re-runs, in seconds. + +.. _patronictl_query_examples: + +Examples +"""""""" + +Run a SQL command as ``postgres`` user, and ask for its password: + +.. code:: bash + + $ patronictl -c postgres0.yml query batman -U postgres --password -c "SELECT now()" + Password: + now + 2023-09-12 18:10:53.228084+00:00 + +Run a SQL command as ``postgres`` user, and take password from ``libpq`` environment variable: + +.. code:: bash + + $ PGPASSWORD=zalando patronictl -c postgres0.yml query batman -U postgres -c "SELECT now()" + now + 2023-09-12 18:11:37.639500+00:00 + +Run a SQL command and print in ``pretty`` format every 2 seconds: + +.. code:: bash + + $ patronictl -c postgres0.yml query batman -c "SELECT now()" --format pretty -W + +----------------------------------+ + | now | + +----------------------------------+ + | 2023-09-12 18:12:16.716235+00:00 | + +----------------------------------+ + +----------------------------------+ + | now | + +----------------------------------+ + | 2023-09-12 18:12:18.732645+00:00 | + +----------------------------------+ + +----------------------------------+ + | now | + +----------------------------------+ + | 2023-09-12 18:12:20.750573+00:00 | + +----------------------------------+ + +Run a SQL command on database ``test`` and print the output in YAML format: + +.. code:: bash + + $ patronictl -c postgres0.yml query batman -d test -c "SELECT now() AS column_1, 'test' AS column_2" --format yaml + - column_1: 2023-09-12 18:14:22.052060+00:00 + column_2: test + +Run a SQL command on member ``postgresql2``: + +.. code:: bash + + $ patronictl -c postgres0.yml query batman -m postgresql2 -c "SHOW port" + port + 5434 + +Run a SQL command on any of the standbys: + +.. code:: bash + + $ patronictl -c postgres0.yml query batman -r replica -c "SHOW port" + port + 5433 + +.. _patronictl_reinit: + +patronictl reinit +^^^^^^^^^^^^^^^^^ + +.. _patronictl_reinit_synopsis: + +Synopsis +"""""""" + +.. code:: text + + reinit + CLUSTER_NAME + [ MEMBER_NAME [, ... ] ] + [ --group CITUS_GROUP ] + [ --wait ] + [ --force ] + +.. _patronictl_reinit_description: + +Description +""""""""""" + +``patronictl reinit`` rebuilds a Postgres standby instance managed by a replica member of the Patroni cluster. + +.. _patronictl_reinit_parameters: + +Parameters +"""""""""" + +``CLUSTER_NAME`` + Name of the Patroni cluster. + +``MEMBER_NAME`` + Name of the replica member for which the Postgres instance will be rebuilt. + + Multiple replica members can be specified. If no members are specified, the command does nothing. + +``--group`` + Rebuild a replica member of the given Citus group. + + ``CITUS_GROUP`` is the ID of the Citus group. + +``--wait`` + Wait until the reinitialization of the Postgres standby node(s) is finished. + +``--force`` + Flag to skip confirmation prompts when rebuilding Postgres standby instances. + + Useful for scripts. + +.. _patronictl_reinit_examples: + +Examples +"""""""" + +Request a rebuild of all replica members of the Patroni cluster and immediately return control to the caller: + +.. code:: bash + + $ patronictl -c postgres0.yml reinit batman postgresql1 postgresql2 --force + + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +-------------+----------------+---------+-----------+----+-----------+ + | postgresql0 | 127.0.0.1:5432 | Leader | running | 5 | | + | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 5 | 0 | + | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 5 | 0 | + +-------------+----------------+---------+-----------+----+-----------+ + Success: reinitialize for member postgresql1 + Success: reinitialize for member postgresql2 + +Request a rebuild of ``postgresql2`` and wait for it to complete: + +.. code:: bash + + $ patronictl -c postgres0.yml reinit batman postgresql2 --wait --force + + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +-------------+----------------+---------+-----------+----+-----------+ + | postgresql0 | 127.0.0.1:5432 | Leader | running | 5 | | + | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 5 | 0 | + | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 5 | 0 | + +-------------+----------------+---------+-----------+----+-----------+ + Success: reinitialize for member postgresql2 + Waiting for reinitialize to complete on: postgresql2 + Reinitialize is completed on: postgresql2 + +.. _patronictl_reload: + +patronictl reload +^^^^^^^^^^^^^^^^^ + +.. _patronictl_reload_synopsis: + +Synopsis +"""""""" + +.. code:: text + + reload + CLUSTER_NAME + [ MEMBER_NAME [, ... ] ] + [ --group CITUS_GROUP ] + [ { -r | --role } { leader | primary | standby-leader | replica | standby | any } ] + [ --force ] + +.. _patronictl_reload_description: + +Description +""""""""""" + +``patronictl reload`` requests a reload of local configuration for one or more Patroni members. + +It also triggers ``pg_ctl reload`` on the managed Postgres instance, even if nothing has changed. + +.. _patronictl_reload_parameters: + +Parameters +"""""""""" + +``CLUSTER_NAME`` + Name of the Patroni cluster. + +``MEMBER_NAME`` + Request a reload of local configuration for the given Patroni member(s). + + Multiple members can be specified. If no members are specified, all of them are considered. + +``--group`` + Request a reload of members of the given Citus group. + + ``CITUS_GROUP`` is the ID of the Citus group. + +``-r`` / ``--role`` + Select members that have the given role. + + Role can be one of: + + - ``leader``: the leader of either a regular Patroni cluster or a standby Patroni cluster; or + - ``primary``: the leader of a regular Patroni cluster; or + - ``standby-leader``: the leader of a standby Patroni cluster; or + - ``replica``: a replica of a Patroni cluster; or + - ``standby``: same as ``replica``; or + - ``any``: any role. Same as omitting this parameter. + +``--force`` + Flag to skip confirmation prompts when requesting a reload of the local configuration. + + Useful for scripts. + +.. _patronictl_reload_examples: + +Examples +"""""""" + +Request a reload of the local configuration of all members of the Patroni cluster: + +.. code:: bash + + $ patronictl -c postgres0.yml reload batman --force + + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +-------------+----------------+---------+-----------+----+-----------+ + | postgresql0 | 127.0.0.1:5432 | Leader | running | 5 | | + | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 5 | 0 | + | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 5 | 0 | + +-------------+----------------+---------+-----------+----+-----------+ + Reload request received for member postgresql0 and will be processed within 10 seconds + Reload request received for member postgresql1 and will be processed within 10 seconds + Reload request received for member postgresql2 and will be processed within 10 seconds + +.. _patronictl_remove: + +patronictl remove +^^^^^^^^^^^^^^^^^ + +.. _patronictl_remove_synopsis: + +Synopsis +"""""""" + +.. code:: text + + remove + CLUSTER_NAME + [ --group CITUS_GROUP ] + [ { -f | --format } { pretty | tsv | json | yaml } ] + +.. _patronictl_remove_description: + +Description +""""""""""" + +``patronictl remove`` removes information of the cluster from the DCS. + +It is an interactive action. + +.. warning:: + This operation will destroy the information of the Patroni cluster from the DCS. + +.. _patronictl_remove_parameters: + +Parameters +"""""""""" + +``CLUSTER_NAME`` + Name of the Patroni cluster. + +``--group`` + Remove information about the Patroni cluster related with the given Citus group. + + ``CITUS_GROUP`` is the ID of the Citus group. + +``-f`` / ``--format`` + How to format the list of members in the output when prompting for confirmation. + + Format can be one of: + + - ``pretty``: prints members as a pretty table; or + - ``tsv``: prints members as tabular information, with columns delimited by ``\t``; or + - ``json``: prints members in JSON format; or + - ``yaml``: prints members in YAML format. + + The default is ``pretty``. + +.. _patronictl_remove_examples: + +Examples +"""""""" + +Remove information about Patroni cluster ``batman`` from the DCS: + +.. code:: bash + + $ patronictl -c postgres0.yml remove batman + + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +-------------+----------------+---------+-----------+----+-----------+ + | postgresql0 | 127.0.0.1:5432 | Leader | running | 5 | | + | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 5 | 0 | + | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 5 | 0 | + +-------------+----------------+---------+-----------+----+-----------+ + Please confirm the cluster name to remove: batman + You are about to remove all information in DCS for batman, please type: "Yes I am aware": Yes I am aware + This cluster currently is healthy. Please specify the leader name to continue: postgresql0 + +.. _patronictl_restart: + +patronictl restart +^^^^^^^^^^^^^^^^^^ + +.. _patronictl_restart_synopsis: + +Synopsis +"""""""" + +.. code:: text + + restart + CLUSTER_NAME + [ MEMBER_NAME [, ...] ] + [ --group CITUS_GROUP ] + [ { -r | --role } { leader | primary | standby-leader | replica | standby | any } ] + [ --any ] + [ --pg-version PG_VERSION ] + [ --pending ] + [ --timeout TIMEOUT ] + [ --scheduled TIMESTAMP ] + [ --force ] + +.. _patronictl_restart_description: + +Description +""""""""""" + +``patronictl restart`` requests a restart of the Postgres instance managed by a member of the Patroni cluster. + +The restart can be performed immediately or scheduled for later. + +.. _patronictl_restart_parameters: + +Parameters +"""""""""" + +``CLUSTER_NAME`` + Name of the Patroni cluster. + +``--group`` + Restart the Patroni cluster related with the given Citus group. + + ``CITUS_GROUP`` is the ID of the Citus group. + +``-r`` / ``--role`` + Choose members that have the given role. + + Role can be one of: + + - ``leader``: the leader of either a regular Patroni cluster or a standby Patroni cluster; or + - ``primary``: the leader of a regular Patroni cluster; or + - ``standby-leader``: the leader of a standby Patroni cluster; or + - ``replica``: a replica of a Patroni cluster; or + - ``standby``: same as ``replica``; or + - ``any``: any role. Same as omitting this parameter. + +``--any`` + Restart a single random node among the ones which match the given filters. + +``--pg-version`` + Select only members which version of the managed Postgres instance is older than the given version. + + ``PG_VERSION`` is the Postgres version to be compared. + +``--pending`` + Select only members which are flagged as ``Pending restart``. + +``timeout`` + Abort the restart if it takes more than the specified timeout, and fail over to a replica if the issue is on the primary. + + ``TIMEOUT`` is the amount of seconds to wait before aborting the restart. + +``--scheduled`` + Schedule a restart to occur at the given timestamp. + + ``TIMESTAMP`` is the timestamp when the restart should occur. Specify it in unambiguous format, preferrably with time zone. You can also use the literal ``now`` for the restart to be executed immediately. + +``--force`` + Flag to skip confirmation prompts when requesting the restart operations. + + Useful for scripts. + +.. _patronictl_restart_examples: + +Examples +"""""""" + +Restart all members of the cluster immediately: + +.. code:: bash + + $ patronictl -c postgres0.yml restart batman --force + + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +-------------+----------------+---------+-----------+----+-----------+ + | postgresql0 | 127.0.0.1:5432 | Leader | running | 6 | | + | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 6 | 0 | + | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 6 | 0 | + +-------------+----------------+---------+-----------+----+-----------+ + Success: restart on member postgresql0 + Success: restart on member postgresql1 + Success: restart on member postgresql2 + +Restart a random member of the cluster immediately: + +.. code:: bash + + $ patronictl -c postgres0.yml restart batman --any --force + + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +-------------+----------------+---------+-----------+----+-----------+ + | postgresql0 | 127.0.0.1:5432 | Leader | running | 6 | | + | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 6 | 0 | + | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 6 | 0 | + +-------------+----------------+---------+-----------+----+-----------+ + Success: restart on member postgresql1 + +Schedule a restart to occur at ``2023-09-13T18:00-03:00``: + +.. code:: bash + + $ patronictl -c postgres0.yml restart batman --scheduled 2023-09-13T18:00-03:00 --force + + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +-------------+----------------+---------+-----------+----+-----------+ + | postgresql0 | 127.0.0.1:5432 | Leader | running | 6 | | + | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 6 | 0 | + | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 6 | 0 | + +-------------+----------------+---------+-----------+----+-----------+ + Success: restart scheduled on member postgresql0 + Success: restart scheduled on member postgresql1 + Success: restart scheduled on member postgresql2 + +.. _patronictl_resume: + +patronictl resume +^^^^^^^^^^^^^^^^^ + +.. _patronictl_resume_synopsis: + +Synopsis +"""""""" + +.. code:: text + + resume + [ CLUSTER_NAME ] + [ --group CITUS_GROUP ] + [ --wait ] + +.. _patronictl_resume_description: + +Description +""""""""""" + +``patronictl resume`` takes the Patroni cluster out of maintenance mode and re-enables automatic failover. + +.. _patronictl_resume_parameters: + +Parameters +"""""""""" + +``CLUSTER_NAME`` + Name of the Patroni cluster. + + If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. + +``--group`` + Resume the given Citus group. + + ``CITUS_GROUP`` is the ID of the Citus group. + + If not given, ``patronictl`` will attempt to fetch that from the ``citus.group`` configuration, if it exists. + +``--wait`` + Wait until all Patroni members are unpaused before returning control to the caller. + +.. _patronictl_resume_examples: + +Examples +"""""""" + +Put the cluster out of maintenance mode: + +.. code:: bash + + $ patronictl -c postgres0.yml resume batman --wait + 'resume' request sent, waiting until it is recognized by all nodes + Success: cluster management is resumed + +.. _patronictl_show_config: + +patronictl show-config +^^^^^^^^^^^^^^^^^^^^^^ + +.. _patronictl_show_config_synopsis: + +Synopsis +"""""""" + +.. code:: text + + show-config + [ CLUSTER_NAME ] + [ --group CITUS_GROUP ] + +.. _patronictl_show_config_description: + +Description +""""""""""" + +``patronictl show-config`` shows the dynamic configuration of the cluster that is stored in the DCS. + +.. _patronictl_show_config_parameters: + +Parameters +"""""""""" + +``CLUSTER_NAME`` + Name of the Patroni cluster. + + If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. + +``--group`` + Show dynamic configuration of the given Citus group. + + ``CITUS_GROUP`` is the ID of the Citus group. + + If not given, ``patronictl`` will attempt to fetch that from the ``citus.group`` configuration, if it exists. + +.. _patronictl_show_config_examples: + +Examples +"""""""" + +Show dynamic configuration of cluster ``batman``: + +.. code:: bash + + $ patronictl -c postgres0.yml show-config batman + loop_wait: 10 + postgresql: + parameters: + max_connections: 250 + pg_hba: + - host replication replicator 127.0.0.1/32 md5 + - host all all 0.0.0.0/0 md5 + use_pg_rewind: true + retry_timeout: 10 + ttl: 30 + +.. _patronictl_switchover: + +patronictl switchover +^^^^^^^^^^^^^^^^^^^^^ + +.. _patronictl_switchover_synopsis: + +Synopsis +"""""""" + +.. code:: text + + switchover + [ CLUSTER_NAME ] + [ --group CITUS_GROUP ] + [ { --leader | --primary } LEADER_NAME ] + --candidate CANDIDATE_NAME + [ --force ] + +.. _patronictl_switchover_description: + +Description +""""""""""" + +``patronictl switchover`` performs a switchover in the cluster. + +It is designed to be used when the cluster is healthy, e.g.: + +- There is a leader; +- There are synchronous standbys available in a synchronous cluster. + +.. note:: + If your cluster is unhealthy you might be interested in ``patronictl failover`` instead. + +.. _patronictl_switchover_parameters: + +Parameters +"""""""""" + +``CLUSTER_NAME`` + Name of the Patroni cluster. + + If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. + +``--group`` + Perform a switchover in the given Citus group. + + ``CITUS_GROUP`` is the ID of the Citus group. + +``--leader`` / ``--primary`` + Indicate who is the leader to be demoted at switchover time. + + ``LEADER_NAME`` should match the name of the current leader in the cluster. + +``--candidate`` + The node to be promoted on switchover, and take the primary role. + + ``CANDIDATE_NAME`` is the name of the node to be promoted. + +``--scheduled`` + Schedule a switchover to occur at the given timestamp. + + ``TIMESTAMP`` is the timestamp when the switchover should occur. Specify it in unambiguous format, preferrably with time zone. You can also use the literal ``now`` for the switchover to be executed immediately. + +``--force`` + Flag to skip confirmation prompts when performing the switchover. + + Useful for scripts. + +.. _patronictl_switchover_examples: + +Examples +"""""""" + +Switch over with node ``postgresql2``: + +.. code:: bash + + $ patronictl -c postgres0.yml switchover batman --leader postgresql0 --candidate postgresql2 --force + Current cluster topology + + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +-------------+----------------+---------+-----------+----+-----------+ + | postgresql0 | 127.0.0.1:5432 | Leader | running | 6 | | + | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 6 | 0 | + | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 6 | 0 | + +-------------+----------------+---------+-----------+----+-----------+ + 2023-09-13 14:15:23.07497 Successfully switched over to "postgresql2" + + Cluster: batman (7277694203142172922) -+---------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +-------------+----------------+---------+---------+----+-----------+ + | postgresql0 | 127.0.0.1:5432 | Replica | stopped | | unknown | + | postgresql1 | 127.0.0.1:5433 | Replica | running | 6 | 0 | + | postgresql2 | 127.0.0.1:5434 | Leader | running | 6 | | + +-------------+----------------+---------+---------+----+-----------+ + +Schedule a switchover between ``postgresql0`` and ``postgresql2`` to occur at ``2023-09-13T18:00:00-03:00``: + +.. code:: bash + + $ patronictl -c postgres0.yml switchover batman --leader postgresql0 --candidate postgresql2 --scheduled 2023-09-13T18:00-03:00 --force + Current cluster topology + + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +-------------+----------------+---------+-----------+----+-----------+ + | postgresql0 | 127.0.0.1:5432 | Leader | running | 8 | | + | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 8 | 0 | + | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 8 | 0 | + +-------------+----------------+---------+-----------+----+-----------+ + 2023-09-13 14:18:11.20661 Switchover scheduled + + Cluster: batman (7277694203142172922) -+-----------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +-------------+----------------+---------+-----------+----+-----------+ + | postgresql0 | 127.0.0.1:5432 | Leader | running | 8 | | + | postgresql1 | 127.0.0.1:5433 | Replica | streaming | 8 | 0 | + | postgresql2 | 127.0.0.1:5434 | Replica | streaming | 8 | 0 | + +-------------+----------------+---------+-----------+----+-----------+ + Switchover scheduled at: 2023-09-13T18:00:00-03:00 + from: postgresql0 + to: postgresql2 + +.. _patronictl_topology: + +patronictl topology +^^^^^^^^^^^^^^^^^^^ + +.. _patronictl_topology_synopsis: + +Synopsis +"""""""" + +.. code:: text + + topology + [ CLUSTER_NAME [, ... ] ] + [ --group CITUS_GROUP ] + [ { -W | { -w | --watch } TIME } ] + +.. _patronictl_topology_description: + +Description +""""""""""" + +``patronictl topology`` shows information about the Patroni cluster and its members with a tree view approach. + +The following information is included in the output: + +``Cluster`` + Name of the Patroni cluster. + + .. note:: + Shown in the table header. + +``System identifier`` + Postgres system identifier. + + .. note:: + Shown in the table header. + +``Member`` + Name of the Patroni member. + + .. note:: + Information in this column is shown as a tree view of members in terms of replication connections. + +``Host`` + Host where the member is located. + +``Role`` + Current role of the member. + + Can be one among: + + * ``Leader``: the current leader of a regular Patroni cluster; or + * ``Standby Leader``: the current leader of a Patroni standby cluster; or + * ``Sync Standby``: a synchronous standby of a Patroni cluster with synchronous mode enabled; or + * ``Replica``: a regular standby of a Patroni cluster. + +``State`` + Current state of Postgres in the Patroni member. + + Some examples among the possible states: + + * ``running``: if Postgres is currently up and running; + * ``streaming``: if a replica and Postgres is currently streaming WALs from the primary node; + * ``in archive recovery``: if a replica and Postgres is currently fetching WALs from the archive; + * ``stopped``: if Postgres had been shut down; + * ``crashed``: if Postgres has crashed. + +``TL`` + Current Postgres timeline in the Patroni member. + +``Lag in MB`` + Amount worth of replication lag in megabytes between the Patroni member and its upstream. + +Besides that, the following information may be included in the output: + +``Group`` + Citus group ID. + + .. note:: + Shown in the table header. + + Only shown if a Citus cluster. + +``Pending restart`` + ``*`` indicates the node needs a restart for some Postgres configuration to take effect. An empty value indicates the node does not require a restart. + + .. note:: + Shown as a member attribute. + + Shown if node requires a restart. + +``Scheduled restart`` + Timestamp at which a restart has been scheduled for the Postgres instance managed by the Patroni member. An empty value indicates there is no scheduled restart for the member. + + .. note:: + Shown as a member attribute. + + Shown if node has a scheduled restart. + +``Tags`` + Contains tags set for the Patroni member. An empty value indicates that either no tags have been configured, or that they have been configured with default values. + + .. note:: + Shown as a member attribute. + + Shown if node has any custom tags, or any default tags with non-default values. + +``Scheduled switchover`` + Timestamp at which a switchover has been scheduled for the Patroni cluster, if any. + + .. note:: + Shown in the table footer. + + Only shown if there is a scheduled switchover. + +``Maintenance mode`` + + If the cluster monitoring is currently paused. + + .. note:: + Shown in the table footer. + + Only shown if the cluster is paused. + +.. _patronictl_topology_parameters: + +Parameters +"""""""""" + +``CLUSTER_NAME`` + Name of the Patroni cluster. + + If not given, ``patronictl`` will attempt to fetch that from the ``scope`` configuration, if it exists. + +``--group`` + Show information about members from the given Citus group. + + ``CITUS_GROUP`` is the ID of the Citus group. + +``-W`` + Automatically refresh information every 2 seconds. + +``-w`` / ``--watch`` + Automatically refresh information at the specified interval. + + ``TIME`` is the interval between refreshes, in seconds. + +.. _patronictl_topology_examples: + +Examples +"""""""" + +Show topology of the cluster ``batman`` -- ``postgresql1`` and ``postgresql2`` are replicating from ``postgresql0``: + +.. code:: bash + + $ patronictl -c postgres0.yml topology batman + + Cluster: batman (7277694203142172922) ---+-----------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +---------------+----------------+---------+-----------+----+-----------+ + | postgresql0 | 127.0.0.1:5432 | Leader | running | 8 | | + | + postgresql1 | 127.0.0.1:5433 | Replica | streaming | 8 | 0 | + | + postgresql2 | 127.0.0.1:5434 | Replica | streaming | 8 | 0 | + +---------------+----------------+---------+-----------+----+-----------+ + +.. _patronictl_version: + +patronictl version +^^^^^^^^^^^^^^^^^^ + +.. _patronictl_version_synopsis: + +Synopsis +"""""""" + +.. code:: text + + version + [ CLUSTER_NAME [, ... ] ] + [ MEMBER_NAME [, ... ] ] + [ --group CITUS_GROUP ] + +.. _patronictl_version_description: + +Description +""""""""""" + +``patronictl version`` gets the version of ``patronictl`` application. Besides that it may also include version information about Patroni clusters and their members. + +.. _patronictl_version_parameters: + +Parameters +"""""""""" + +``CLUSTER_NAME`` + Name of the Patroni cluster. + +``MEMBER_NAME`` + Name of the member of the Patroni cluster. + +``--group`` + Consider a Patroni cluster with the given Citus group. + + ``CITUS_GROUP`` is the ID of the Citus group. + +.. _patronictl_version_examples: + +Examples +"""""""" + +Get version of ``patronictl`` only: + +.. code:: bash + + $ patronictl -c postgres0.yml version + patronictl version 3.1.0 + +Get version of ``patronictl`` and of all members of cluster ``batman``: + +.. code:: bash + + $ patronictl -c postgres0.yml version batman + patronictl version 3.1.0 + + postgresql0: Patroni 3.1.0 PostgreSQL 15.2 + postgresql1: Patroni 3.1.0 PostgreSQL 15.2 + postgresql2: Patroni 3.1.0 PostgreSQL 15.2 + +Get version of ``patronictl`` and of members ``postgresql1`` and ``postgresql2`` of cluster ``batman``: + +.. code:: bash + + $ patronictl -c postgres0.yml version batman postgresql1 postgresql2 + patronictl version 3.1.0 + + postgresql1: Patroni 3.1.0 PostgreSQL 15.2 + postgresql2: Patroni 3.1.0 PostgreSQL 15.2 diff --git a/docs/pause.rst b/docs/pause.rst index 60a01a4c..9a74a2e0 100644 --- a/docs/pause.rst +++ b/docs/pause.rst @@ -32,6 +32,6 @@ When Patroni runs in a paused mode, it does not change the state of PostgreSQL, User guide ---------- -``patronictl`` supports ``pause`` and ``resume`` commands. +``patronictl`` supports :ref:`pause ` and :ref:`resume ` commands. One can also issue a ``PATCH`` request to the ``{namespace}/{cluster}/config`` key with ``{"pause": true/false/null}`` diff --git a/docs/replica_bootstrap.rst b/docs/replica_bootstrap.rst index 42a7d3b0..5ae53103 100644 --- a/docs/replica_bootstrap.rst +++ b/docs/replica_bootstrap.rst @@ -191,7 +191,7 @@ There is no further relationship between the standby cluster and the primary cluster it replicates from, in particular, they must not share the same DCS scope if they use the same DCS. They do not know anything else from each other apart from replication information. Also, the standby cluster is not being -displayed in ``patronictl list`` or ``patronictl topology`` output on the +displayed in :ref:`patronictl_list` or :ref:`patronictl_topology` output on the primary cluster. For the sake of flexibility, you can specify methods of creating a replica and diff --git a/docs/rest_api.rst b/docs/rest_api.rst index 52d417ec..00272100 100644 --- a/docs/rest_api.rst +++ b/docs/rest_api.rst @@ -3,7 +3,7 @@ Patroni REST API ================ -Patroni has a rich REST API, which is used by Patroni itself during the leader race, by the ``patronictl`` tool in order to perform failovers/switchovers/reinitialize/restarts/reloads, by HAProxy or any other kind of load balancer to perform HTTP health checks, and of course could also be used for monitoring. Below you will find the list of Patroni REST API endpoints. +Patroni has a rich REST API, which is used by Patroni itself during the leader race, by the :ref:`patronictl` tool in order to perform failovers/switchovers/reinitialize/restarts/reloads, by HAProxy or any other kind of load balancer to perform HTTP health checks, and of course could also be used for monitoring. Below you will find the list of Patroni REST API endpoints. Health check endpoints ---------------------- @@ -619,9 +619,9 @@ In the JSON body of the ``POST`` request you must specify the ``candidate`` fiel :ref:`Be very careful ` when using this endpoint, as this can cause data loss in certain situations. In most cases, :ref:`the switchover endpoint ` satisfies the administrator's needs. -``POST /switchover`` and ``POST /failover`` endpoints are used by ``patronictl switchover`` and ``patronictl failover``, respectively. +``POST /switchover`` and ``POST /failover`` endpoints are used by :ref:`patronictl_switchover` and :ref:`patronictl_failover`, respectively. -``DELETE /switchover`` is used by ``patronictl flush switchover``. +``DELETE /switchover`` is used by :ref:`patronictl flush cluster-name switchover `. .. list-table:: Failover/Switchover comparison :widths: 25 25 25 @@ -680,15 +680,15 @@ Restart endpoint - ``DELETE /restart``: delete the scheduled restart -``POST /restart`` and ``DELETE /restart`` endpoints are used by ``patronictl restart`` and ``patronictl flush restart`` respectively. +``POST /restart`` and ``DELETE /restart`` endpoints are used by :ref:`patronictl_restart` and :ref:`patronictl flush cluster-name restart ` respectively. Reload endpoint --------------- -The ``POST /reload`` call will order Patroni to re-read and apply the configuration file. This is the equivalent of sending the ``SIGHUP`` signal to the Patroni process. In case you changed some of the Postgres parameters which require a restart (like **shared_buffers**), you still have to explicitly do the restart of Postgres by either calling the ``POST /restart`` endpoint or with the help of ``patronictl restart``. +The ``POST /reload`` call will order Patroni to re-read and apply the configuration file. This is the equivalent of sending the ``SIGHUP`` signal to the Patroni process. In case you changed some of the Postgres parameters which require a restart (like **shared_buffers**), you still have to explicitly do the restart of Postgres by either calling the ``POST /restart`` endpoint or with the help of :ref:`patronictl_restart`. -The reload endpoint is used by ``patronictl reload``. +The reload endpoint is used by :ref:`patronictl_reload`. Reinitialize endpoint @@ -698,4 +698,4 @@ Reinitialize endpoint The call might fail if Patroni is in a loop trying to recover (restart) a failed Postgres. In order to overcome this problem one can specify ``{"force":true}`` in the request body. -The reinitialize endpoint is used by ``patronictl reinit``. +The reinitialize endpoint is used by :ref:`patronictl_reinit`. diff --git a/docs/security.rst b/docs/security.rst index cddefe0c..24af168e 100644 --- a/docs/security.rst +++ b/docs/security.rst @@ -9,7 +9,7 @@ A Patroni cluster has two interfaces to be protected from unauthorized access: t Protecting DCS ============== -Patroni and patronictl both store and retrieve data to/from the DCS. +Patroni and :ref:`patronictl` both store and retrieve data to/from the DCS. Despite DCS doesn't contain any sensitive information, it allows changing some of Patroni/Postgres configuration. Therefore the very first thing that should be protected is DCS itself. @@ -22,7 +22,7 @@ Protecting the REST API Protecting the REST API is a more complicated task. -The Patroni REST API is used by Patroni itself during the leader race, by the ``patronictl`` tool in order to perform failovers/switchovers/reinitialize/restarts/reloads, by HAProxy or any other kind of load balancer to perform HTTP health checks, and of course could also be used for monitoring. +The Patroni REST API is used by Patroni itself during the leader race, by the :ref:`patronictl` tool in order to perform failovers/switchovers/reinitialize/restarts/reloads, by HAProxy or any other kind of load balancer to perform HTTP health checks, and of course could also be used for monitoring. From the point of view of security, REST API contains safe (``GET`` requests, only retrieve information) and unsafe (``PUT``, ``POST``, ``PATCH`` and ``DELETE`` requests, change the state of nodes) endpoints. @@ -32,6 +32,6 @@ When TLS for the REST API is enabled and a PKI is established, mutual authentica The ``restapi`` section parameters enable TLS client authentication to the server. Depending on the value of the ``verify_client`` parameter, the API server requires a successful client certificate verification for both safe and unsafe API calls (``verify_client: required``), or only for unsafe API calls (``verify_client: optional``), or for no API calls (``verify_client: none``). -The ``ctl`` section parameters enable TLS server authentication to the client (the ``patronictl`` tool which uses the same config as patroni). Set ``insecure: true`` to disable the server certificate verification by the client. See :ref:`settings ` for a detailed description of the TLS client parameters. +The ``ctl`` section parameters enable TLS server authentication to the client (the :ref:`patronictl` tool which uses the same config as patroni). Set ``insecure: true`` to disable the server certificate verification by the client. See :ref:`settings ` for a detailed description of the TLS client parameters. Protecting the PostgreSQL database proper from unauthorized access is beyond the scope of this document and is covered in https://www.postgresql.org/docs/current/client-authentication.html diff --git a/docs/yaml_configuration.rst b/docs/yaml_configuration.rst index d9283192..4c2ed7c0 100644 --- a/docs/yaml_configuration.rst +++ b/docs/yaml_configuration.rst @@ -34,7 +34,7 @@ Bootstrap configuration .. note:: Once Patroni has initialized the cluster for the first time and settings have been stored in the DCS, all future changes to the ``bootstrap.dcs`` section of the YAML configuration will not take any effect! If you want to change - them please use either ``patronictl edit-config`` or the Patroni :ref:`REST API `. + them please use either :ref:`patronictl_edit_config` or the Patroni :ref:`REST API `. - **bootstrap**: @@ -366,10 +366,10 @@ CTL - **authentication**: - - **username**: Basic-auth username for accessing protected REST API endpoints. If not provided patronictl will use the value provided for REST API "username" parameter. - - **password**: Basic-auth password for accessing protected REST API endpoints. If not provided patronictl will use the value provided for REST API "password" parameter. + - **username**: Basic-auth username for accessing protected REST API endpoints. If not provided :ref:`patronictl` will use the value provided for REST API "username" parameter. + - **password**: Basic-auth password for accessing protected REST API endpoints. If not provided :ref:`patronictl` will use the value provided for REST API "password" parameter. - **insecure**: Allow connections to REST API without verifying SSL certs. - - **cacert**: Specifies the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs. If not provided patronictl will use the value provided for REST API "cafile" parameter. + - **cacert**: Specifies the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs. If not provided :ref:`patronictl` will use the value provided for REST API "cafile" parameter. - **certfile**: Specifies the file with the client certificate in the PEM format. - **keyfile**: Specifies the file with the client secret key in the PEM format. - **keyfile\_password**: Specifies a password for decrypting the client keyfile. @@ -397,4 +397,4 @@ In addition to these predefined tags, you can also add your own ones: - **key3**: ``1.4`` - **key4**: ``"RandomString"`` -Tags are visible in the :ref:`REST API ` and ``patronictl list`` You can also check for an instance health using these tags. If the tag isn't defined for an instance, or if the respective value doesn't match the querying value, it will return HTTP Status Code 503. +Tags are visible in the :ref:`REST API ` and :ref:`patronictl_list` You can also check for an instance health using these tags. If the tag isn't defined for an instance, or if the respective value doesn't match the querying value, it will return HTTP Status Code 503. From 9283ebda6487f0ac12a30e068913d34e95567c58 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 4 Oct 2023 11:44:57 +0200 Subject: [PATCH 07/11] Enforce loop_wait/retry_timeout/ttl rule (#2869) * hard-code minimal possible values * make adjustments if values are lower or if the rule is violated and show warnings * update documentation --- docs/dynamic_configuration.rst | 15 +++++++-- patroni/config.py | 61 ++++++++++++++++++++++++++++++++++ tests/test_config.py | 24 +++++++++++++ 3 files changed, 97 insertions(+), 3 deletions(-) diff --git a/docs/dynamic_configuration.rst b/docs/dynamic_configuration.rst index 285bcab3..a16a2a40 100644 --- a/docs/dynamic_configuration.rst +++ b/docs/dynamic_configuration.rst @@ -8,9 +8,18 @@ Dynamic configuration is stored in the DCS (Distributed Configuration Store) and In order to change the dynamic configuration you can use either :ref:`patronictl_edit_config` tool or Patroni :ref:`REST API `. -- **loop\_wait**: the number of seconds the loop will sleep. Default value: 10 -- **ttl**: the TTL to acquire the leader lock (in seconds). Think of it as the length of time before initiation of the automatic failover process. Default value: 30 -- **retry\_timeout**: timeout for DCS and PostgreSQL operation retries (in seconds). DCS or network issues shorter than this will not cause Patroni to demote the leader. Default value: 10 +- **loop\_wait**: the number of seconds the loop will sleep. Default value: 10, minimum possible value: 1 +- **ttl**: the TTL to acquire the leader lock (in seconds). Think of it as the length of time before initiation of the automatic failover process. Default value: 30, minimum possible value: 20 +- **retry\_timeout**: timeout for DCS and PostgreSQL operation retries (in seconds). DCS or network issues shorter than this will not cause Patroni to demote the leader. Default value: 10, minimum possible value: 3 + +.. warning:: + when changing values of **loop_wait**, **retry_timeout**, or **ttl** you have to follow the rule: + + .. code-block:: python + + loop_wait + 2 * retry_timeout <= ttl + + - **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election. - **maximum\_lag\_on\_syncnode**: the maximum bytes a synchronous follower may lag before it is considered as an unhealthy candidate and swapped by healthy asynchronous follower. Patroni utilize the max replica lsn if there is more than one follower, otherwise it will use leader's current wal lsn. Default is -1, Patroni will not take action to swap synchronous unhealthy follower when the value is set to 0 or below. Please set the value high enough so Patroni won't swap synchrounous follower fequently during high transaction volume. - **max\_timelines\_history**: maximum number of timeline history items kept in DCS. Default value: 0. When set to 0, it keeps the full history in DCS. diff --git a/patroni/config.py b/patroni/config.py index fee2147a..a0384504 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -400,6 +400,66 @@ class Config(object): except Exception: logger.error('Can not remove temporary file %s', tmpfile) + def __get_and_maybe_adjust_int_value(self, config: Dict[str, Any], param: str, min_value: int) -> int: + """Get, validate and maybe adjust a *param* integer value from the *config* :class:`dict`. + + .. note: + If the value is smaller than provided *min_value* we update the *config*. + + This method may raise an exception if value isn't :class:`int` or cannot be casted to :class:`int`. + + :param config: :class:`dict` object with new global configuration. + :param param: name of the configuration parameter we want to read/validate/adjust. + :param min_value: the minimum possible value that a given *param* could have. + + :returns: an integer value which corresponds to a provided *param*. + """ + value = int(config.get(param, self.__DEFAULT_CONFIG[param])) + if value < min_value: + logger.warning("%s=%d can't be smaller than %d, adjusting...", param, value, min_value) + value = config[param] = min_value + return value + + def _validate_and_adjust_timeouts(self, config: Dict[str, Any]) -> None: + """Validate and adjust ``loop_wait``, ``retry_timeout``, and ``ttl`` values if necessary. + + Minimum values: + + * ``loop_wait``: 1 second; + * ``retry_timeout``: 3 seconds. + * ``ttl``: 20 seconds; + + Maximum values: + In case if values don't fulfill the following rule, ``retry_timeout`` and ``loop_wait`` + are reduced so that the rule is fulfilled: + + .. code-block:: python + + loop_wait + 2 * retry_timeout <= ttl + + .. note: + We prefer to reduce ``loop_wait`` and will reduce ``retry_timeout`` only if ``loop_wait`` + is already set to a minimal possible value. + + :param config: :class:`dict` object with new global configuration. + """ + + min_loop_wait = 1 + loop_wait = self. __get_and_maybe_adjust_int_value(config, 'loop_wait', min_loop_wait) + retry_timeout = self. __get_and_maybe_adjust_int_value(config, 'retry_timeout', 3) + ttl = self. __get_and_maybe_adjust_int_value(config, 'ttl', 20) + + if min_loop_wait + 2 * retry_timeout > ttl: + config['loop_wait'] = min_loop_wait + config['retry_timeout'] = (ttl - min_loop_wait) // 2 + logger.warning('Violated the rule "loop_wait + 2*retry_timeout <= ttl", where ttl=%d. ' + 'Adjusting loop_wait from %d to %d and retry_timeout from %d to %d', + ttl, loop_wait, min_loop_wait, retry_timeout, config['retry_timeout']) + elif loop_wait + 2 * retry_timeout > ttl: + config['loop_wait'] = ttl - 2 * retry_timeout + logger.warning('Violated the rule "loop_wait + 2*retry_timeout <= ttl", where ttl=%d and retry_timeout=%d.' + ' Adjusting loop_wait from %d to %d', ttl, retry_timeout, loop_wait, config['loop_wait']) + # configuration could be either ClusterConfig or dict def set_dynamic_configuration(self, configuration: Union[ClusterConfig, Dict[str, Any]]) -> bool: """Set dynamic configuration values with given *configuration*. @@ -417,6 +477,7 @@ class Config(object): if not deep_compare(self._dynamic_configuration, configuration): try: + self._validate_and_adjust_timeouts(configuration) self.__effective_configuration = self._build_effective_configuration(configuration, self._local_configuration) self._dynamic_configuration = configuration diff --git a/tests/test_config.py b/tests/test_config.py index f0a780bc..8515476a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -173,3 +173,27 @@ class TestConfig(unittest.TestCase): input_params['max_connections'] = 10 expected_params.pop('max_connections') self.assertEqual(self.config._process_postgresql_parameters(input_params), expected_params) + + def test__validate_and_adjust_timeouts(self): + with patch('patroni.config.logger.warning') as mock_logger: + self.config._validate_and_adjust_timeouts({'ttl': 15}) + self.assertEqual(mock_logger.call_args_list[0][0], + ("%s=%d can't be smaller than %d, adjusting...", 'ttl', 15, 20)) + with patch('patroni.config.logger.warning') as mock_logger: + self.config._validate_and_adjust_timeouts({'loop_wait': 0}) + self.assertEqual(mock_logger.call_args_list[0][0], + ("%s=%d can't be smaller than %d, adjusting...", 'loop_wait', 0, 1)) + with patch('patroni.config.logger.warning') as mock_logger: + self.config._validate_and_adjust_timeouts({'retry_timeout': 1}) + self.assertEqual(mock_logger.call_args_list[0][0], + ("%s=%d can't be smaller than %d, adjusting...", 'retry_timeout', 1, 3)) + with patch('patroni.config.logger.warning') as mock_logger: + self.config._validate_and_adjust_timeouts({'ttl': 20, 'loop_wait': 11, 'retry_timeout': 5}) + self.assertEqual(mock_logger.call_args_list[0][0], + ('Violated the rule "loop_wait + 2*retry_timeout <= ttl", where ttl=%d ' + 'and retry_timeout=%d. Adjusting loop_wait from %d to %d', 20, 5, 11, 10)) + with patch('patroni.config.logger.warning') as mock_logger: + self.config._validate_and_adjust_timeouts({'ttl': 20, 'loop_wait': 10, 'retry_timeout': 10}) + self.assertEqual(mock_logger.call_args_list[0][0], + ('Violated the rule "loop_wait + 2*retry_timeout <= ttl", where ttl=%d. Adjusting' + ' loop_wait from %d to %d and retry_timeout from %d to %d', 20, 10, 1, 10, 9)) From efacc6c16b8c0af6e258ee353b8913c120157b68 Mon Sep 17 00:00:00 2001 From: Polina Bungina <27892524+hughcapet@users.noreply.github.com> Date: Fri, 6 Oct 2023 10:21:37 +0200 Subject: [PATCH 08/11] Ignore synchronous_mode setting in a standby cluster (#2896) is_synchronous_mode() should always return False in standby clusters --- features/standby_cluster.feature | 27 +++++++++++++++++---------- features/steps/standby_cluster.py | 1 + patroni/config.py | 4 ++-- tests/test_config.py | 7 ++++++- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/features/standby_cluster.feature b/features/standby_cluster.feature index 7ff07679..ac6bbb85 100644 --- a/features/standby_cluster.feature +++ b/features/standby_cluster.feature @@ -53,15 +53,22 @@ Feature: standby cluster And I receive a response replication_state streaming And postgres1 does not have a logical replication slot named test_logical - Scenario: check failover - When I kill postgres1 - And I kill postmaster on postgres1 - Then postgres2 is replicating from postgres0 after 32 seconds - When I issue a GET request to http://127.0.0.1:8010/primary - Then I receive a response code 503 - And I sleep for 3 seconds - When I issue a GET request to http://127.0.0.1:8010/standby_leader + Scenario: check switchover + When I run patronictl.py switchover batman1 --force + And I issue a GET request to http://127.0.0.1:8010/standby_leader Then I receive a response code 200 And I receive a response role standby_leader - And replication works from postgres0 to postgres2 after 15 seconds - And there is a postgres2_cb.log with "on_start replica batman1\non_role_change standby_leader batman1" in postgres2 data directory + And postgres1 is replicating from postgres2 after 32 seconds + + Scenario: check failover + When I kill postgres2 + And I kill postmaster on postgres2 + Then postgres1 is replicating from postgres0 after 32 seconds + When I issue a GET request to http://127.0.0.1:8009/primary + Then I receive a response code 503 + And I sleep for 3 seconds + When I issue a GET request to http://127.0.0.1:8009/standby_leader + Then I receive a response code 200 + And I receive a response role standby_leader + And replication works from postgres0 to postgres1 after 15 seconds + And there is a postgres1_cb.log with "on_start replica batman1\non_role_change standby_leader batman1" in postgres1 data directory diff --git a/features/steps/standby_cluster.py b/features/steps/standby_cluster.py index 17d635b8..a428130c 100644 --- a/features/steps/standby_cluster.py +++ b/features/steps/standby_cluster.py @@ -34,6 +34,7 @@ def start_patroni_standby_cluster(context, name, cluster_name, name2): "ttl": 20, "loop_wait": 2, "retry_timeout": 5, + "synchronous_mode": True, # should be completely ignored "standby_cluster": { "host": "localhost", "port": port, diff --git a/patroni/config.py b/patroni/config.py index a0384504..5779aaac 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -96,8 +96,8 @@ class GlobalConfig(object): @property def is_synchronous_mode(self) -> bool: - """``True`` if synchronous replication is requested.""" - return self.check_mode('synchronous_mode') + """``True`` if synchronous replication is requested and it is not a standby cluster config.""" + return self.check_mode('synchronous_mode') and not self.is_standby_cluster @property def is_synchronous_mode_strict(self) -> bool: diff --git a/tests/test_config.py b/tests/test_config.py index 8515476a..2b21f589 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,7 +5,7 @@ import io from copy import deepcopy from mock import MagicMock, Mock, patch -from patroni.config import Config, ConfigParseError +from patroni.config import Config, ConfigParseError, GlobalConfig class TestConfig(unittest.TestCase): @@ -197,3 +197,8 @@ class TestConfig(unittest.TestCase): self.assertEqual(mock_logger.call_args_list[0][0], ('Violated the rule "loop_wait + 2*retry_timeout <= ttl", where ttl=%d. Adjusting' ' loop_wait from %d to %d and retry_timeout from %d to %d', 20, 10, 1, 10, 9)) + + def test_global_config_is_synchronous_mode(self): + # we should ignore synchronous_mode setting in a standby cluster + config = {'standby_cluster': {'host': 'some_host'}, 'synchronous_mode': True} + self.assertFalse(GlobalConfig(config).is_synchronous_mode) From 28a604983be59848637c3157de92ea38ee7bb6ec Mon Sep 17 00:00:00 2001 From: Israel Date: Fri, 6 Oct 2023 05:48:55 -0300 Subject: [PATCH 09/11] Enhancement to `tox` behave tests (#2889) * Add `etcd3` as a DCS option for behave tests in `tox.ini` Currently behave tests run through `tox` accept only `etcd` as a DCS. This commit adds the option of using `etcd3` too. * Add JSON report to `tox` behave tests This commit adds a JSON report when running behave tests through `tox`. That makes it easier to parse the results. --------- Signed-off-by: Israel Barth Rubio --- tox.ini | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index bf66caef..21688937 100644 --- a/tox.ini +++ b/tox.ini @@ -125,10 +125,11 @@ commands = --file features/Dockerfile allowlist_externals = docker -[testenv:pg{12,13,14,15,16}-docker-behave-{etcd}-{lin,mac}] +[testenv:pg{12,13,14,15,16}-docker-behave-{etcd,etcd3}-{lin,mac}] description = Run behaviour tests in patroni-dev docker container setenv = etcd: DCS=etcd + etcd3: DCS=etcd3 {[common]postgres_matrix} CONTAINER_NAME = tox-{env_name}-{env:PYTHONHASHSEED} labels = @@ -149,7 +150,7 @@ commands = --tty \ {env:PATRONI_DEV_IMAGE:patroni-dev:{env:PG_MAJOR}} \ tox run -x 'tox.env_list=py{[common]python_matrix}-behave-{env:DCS}-lin' \ - -- --format plain {posargs} + -- {posargs} allowlist_externals = docker @@ -159,7 +160,7 @@ platform = ; win: win32 mac: darwin -[testenv:py{36,38,39,310,311}-behave-{etcd}-{lin,win,mac}] +[testenv:py{36,38,39,310,311}-behave-{etcd,etcd3}-{lin,win,mac}] description = Run behaviour tests (locally with tox) deps = -r requirements.txt @@ -167,11 +168,15 @@ deps = coverage {[common]psycopg_deps} setenv = - DCS = {env:DCS:etcd} + etcd: DCS = {env:DCS:etcd} + etcd3: DCS = {env:DCS:etcd3} passenv = ETCD_UNSUPPORTED_ARCH commands = - python3 -m behave {posargs} + python3 -m behave --format json --format plain --outfile result.json {posargs} + mv result.json features/output +allowlist_externals = + mv platform = {[common]platforms} From e19a8730eaa4bfbdd3ad59a2f82bbac3ed655493 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 9 Oct 2023 10:43:43 +0200 Subject: [PATCH 10/11] Take IP from the pod if `kubernetes.pod_ip` is missing (#2895) It used to work before #2652 Besides that fix a couple of more problems: - make sure `_patch_or_create()` method isn't instantiating the `k8s_client.V1ConfigMap` object instead of `k8s_client.V1Endpoints` for non leader endpoints. The only reason it worked is that the JSON serialization for both object types is the same and doesn't include the object type name. - `attempt_to_acquire_leader()` should immediately put the IP address of the primary to the leader endpoint. It didn't happen because of the oversight in the https://github.com/zalando/patroni/pull/1820. --- patroni/dcs/kubernetes.py | 32 ++++++++++++++++++++++++++------ tests/test_kubernetes.py | 17 ++++++++++++++++- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/patroni/dcs/kubernetes.py b/patroni/dcs/kubernetes.py index d4158368..aee87bd3 100644 --- a/patroni/dcs/kubernetes.py +++ b/patroni/dcs/kubernetes.py @@ -771,8 +771,7 @@ class Kubernetes(AbstractDCS): except k8s_config.ConfigException: k8s_config.load_kube_config(context=config.get('context', 'kind-kind')) - pod_ip = config.get('pod_ip') - self.__ips: List[str] = [] if self._ctl or not isinstance(pod_ip, str) else [pod_ip] + self.__ips: List[str] = [] if self._ctl else [config.get('pod_ip', '')] self.__ports: List[K8sObject] = [] ports: List[Dict[str, Any]] = config.get('ports', [{}]) for p in ports: @@ -1059,6 +1058,27 @@ class Kubernetes(AbstractDCS): def _patch_or_create(self, name: str, annotations: Dict[str, Any], resource_version: Optional[str] = None, patch: bool = False, retry: Optional[Callable[..., Any]] = None, ips: Optional[List[str]] = None) -> K8sObject: + """Patch or create K8s object, Endpoint or ConfigMap. + + :param name: the name of the object. + :param annotations: mapping of annotations that we want to create/update. + :param resource_version: object should be updated only if the ``resource_version`` matches provided value. + :param patch: ``True`` if we know in advance that the object already exists and we should patch it. + :param retry: a callable that will take care of retries + :param ips: IP address that we want to put to the subsets of the endpoint. Could have following values: + + * ``None`` - when we don't need to touch subset; + * ``[]`` - to set subsets to the empty list, when :meth:`delete_leader` method is called; + + * ``['ip.add.re.ss']`` - when we want to make sure that the subsets of the leader endpoint + contains the IP address of the leader, that we get from the ``kubernetes.pod_ip``; + + * ``['']`` - when we want to make sure that the subsets of the leader endpoint contains the IP + address of the leader, but ``kubernetes.pod_ip`` configuration is missing. In this case we will + try to take the IP address of the Pod which name matches ``name`` from the config file. + + :returns: the new :class:`V1Endpoints` or :class:`V1ConfigMap` object, that was created or updated. + """ metadata = {'namespace': self._namespace, 'name': name, 'labels': self._labels, 'annotations': annotations} if patch or resource_version: if resource_version is not None: @@ -1071,9 +1091,10 @@ class Kubernetes(AbstractDCS): metadata['annotations'] = {k: v for k, v in annotations.items() if v is not None} metadata = k8s_client.V1ObjectMeta(**metadata) - if ips is not None and self._api.use_endpoints: + if self._api.use_endpoints: endpoints = {'metadata': metadata} - self._map_subsets(endpoints, ips) + if ips is not None: + self._map_subsets(endpoints, ips) body = k8s_client.V1Endpoints(**endpoints) else: body = k8s_client.V1ConfigMap(metadata=metadata) @@ -1222,11 +1243,10 @@ class Kubernetes(AbstractDCS): else: annotations['acquireTime'] = self._leader_observed_record.get('acquireTime') or now annotations['transitions'] = str(transitions) - ips: Optional[List[str]] = [] if self._api.use_endpoints else None try: ret = bool(self._patch_or_create(self.leader_path, annotations, - self._leader_resource_version, retry=self.retry, ips=ips)) + self._leader_resource_version, retry=self.retry, ips=self.__ips)) except k8s_client.rest.ApiException as e: if e.status == 409 and self._leader_resource_version: # Conflict in resource_version # Terminate watchers, it could be a sign that K8s API is in a failed state diff --git a/tests/test_kubernetes.py b/tests/test_kubernetes.py index 4f9f418c..b6db7fb3 100644 --- a/tests/test_kubernetes.py +++ b/tests/test_kubernetes.py @@ -63,7 +63,7 @@ def mock_list_namespaced_pod(*args, **kwargs): metadata = k8s_client.V1ObjectMeta(resource_version='1', labels={'f': 'b', Kubernetes._CITUS_LABEL: '1'}, name='p-0', annotations={'status': '{}'}, uid='964dfeae-e79b-4476-8a5a-1920b5c2a69d') - status = k8s_client.V1PodStatus(pod_ip='10.0.0.0') + status = k8s_client.V1PodStatus(pod_ip='10.0.0.1') spec = k8s_client.V1PodSpec(hostname='p-0', node_name='kind-control-plane', containers=[]) items = [k8s_client.V1Pod(metadata=metadata, status=status, spec=spec)] return k8s_client.V1PodList(items=items, kind='PodList') @@ -356,6 +356,20 @@ class TestKubernetesConfigMaps(BaseTestKubernetes): mock_warning.assert_called_once() +class TestKubernetesEndpointsNoPodIP(BaseTestKubernetes): + @patch.object(k8s_client.CoreV1Api, 'list_namespaced_endpoints', mock_list_namespaced_endpoints, create=True) + def setUp(self, config=None): + super(TestKubernetesEndpointsNoPodIP, self).setUp({'use_endpoints': True}) + + @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_endpoints', create=True) + def test_update_leader(self, mock_patch_namespaced_endpoints): + leader = self.k.get_cluster().leader + self.assertIsNotNone(self.k.update_leader(leader, '123', failsafe={'foo': 'bar'})) + args = mock_patch_namespaced_endpoints.call_args[0] + self.assertEqual(args[2].subsets[0].addresses[0].target_ref.resource_version, '1') + self.assertEqual(args[2].subsets[0].addresses[0].ip, '10.0.0.1') + + class TestKubernetesEndpoints(BaseTestKubernetes): @patch.object(k8s_client.CoreV1Api, 'list_namespaced_endpoints', mock_list_namespaced_endpoints, create=True) @@ -368,6 +382,7 @@ class TestKubernetesEndpoints(BaseTestKubernetes): self.assertIsNotNone(self.k.update_leader(leader, '123', failsafe={'foo': 'bar'})) args = mock_patch_namespaced_endpoints.call_args[0] self.assertEqual(args[2].subsets[0].addresses[0].target_ref.resource_version, '10') + self.assertEqual(args[2].subsets[0].addresses[0].ip, '10.0.0.0') self.k._kinds._object_cache['test'].subsets[:] = [] self.assertIsNotNone(self.k.update_leader(leader, '123')) self.k._kinds._object_cache['test'].metadata.annotations['leader'] = 'p-1' From 9b8c40a6e105c16f998a4a487f707b200d4f26fa Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 10 Oct 2023 09:54:24 +0200 Subject: [PATCH 11/11] Start thread that will handle SIGCHLD for on_reload callback (#2898) Close #2897 --- patroni/postgresql/callback_executor.py | 4 +++- tests/test_callback_executor.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/patroni/postgresql/callback_executor.py b/patroni/postgresql/callback_executor.py index 3ae073fd..fa645b86 100644 --- a/patroni/postgresql/callback_executor.py +++ b/patroni/postgresql/callback_executor.py @@ -30,7 +30,9 @@ class OnReloadExecutor(CancellableSubprocess): self.cancel(kill=True) self._kill_children() with self._lock: - self._start_process(cmd, close_fds=True) + started = self._start_process(cmd, close_fds=True) + if started and self._process is not None: + Thread(target=self._process.wait).start() class CallbackExecutor(CancellableExecutor, Thread): diff --git a/tests/test_callback_executor.py b/tests/test_callback_executor.py index df2556b9..51c915d9 100644 --- a/tests/test_callback_executor.py +++ b/tests/test_callback_executor.py @@ -35,5 +35,6 @@ class TestCallbackExecutor(unittest.TestCase): ce._invoke_excepthook = Mock() self.assertIsNone(ce.call(callback)) + mock_popen.side_effect = [Mock()] self.assertIsNone(ce.call(['test.sh', 'on_reload', 'replica', 'foo'])) ce.join()