From 6c06f5cc9650c57f9e9c4af7230784efd57af9ca Mon Sep 17 00:00:00 2001 From: Polina Bungina <27892524+hughcapet@users.noreply.github.com> Date: Wed, 25 Oct 2023 14:20:17 +0200 Subject: [PATCH 01/19] Add initial docs for patroni --validate/generate config (#2929) For now it will sit in the section about the Patroni configuration. We can later move it to (or reference from) a new section where all the functionality of the `patroni` executable will be described. --- docs/existing_data.rst | 2 +- docs/patroni_configuration.rst | 104 +++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 1 deletion(-) diff --git a/docs/existing_data.rst b/docs/existing_data.rst index 442b0b1c..2aa3ad26 100644 --- a/docs/existing_data.rst +++ b/docs/existing_data.rst @@ -36,7 +36,7 @@ You can find below an overview of steps for converting an existing Postgres clus #. If you are running Postgres through systemd, then disable the Postgres systemd unit. This is performed as Patroni manages starting and stopping the Postgres daemon. - #. Create a YAML configuration file for Patroni. + #. Create a YAML configuration file for Patroni. You can use :ref:`Patroni configuration generation and validation tooling ` for that. * **Note (specific for the primary node):** If you have replication slots being used for replication between cluster members, then it is recommended that you enable ``use_slots`` and configure the existing replication slots as permanent via the ``slots`` configuration item. Be aware that Patroni automatically creates replication slots for replication between members, and drops replication slots that it does not recognize, when ``use_slots`` is enabled. The idea of using permanent slots here is to allow your existing slots to persist while the migration to Patroni is in progress. See :ref:`YAML Configuration Settings ` for details. diff --git a/docs/patroni_configuration.rst b/docs/patroni_configuration.rst index e18c118e..1d6c0777 100644 --- a/docs/patroni_configuration.rst +++ b/docs/patroni_configuration.rst @@ -143,3 +143,107 @@ Also the following Patroni configuration options **can be changed only dynamical Upon changing these options, Patroni will read the relevant section of the configuration stored in DCS and change its run-time values. Patroni nodes are dumping the state of the DCS options to disk upon for every change of the configuration into the file ``patroni.dynamic.json`` located in the Postgres data directory. Only the leader is allowed to restore these options from the on-disk dump if these are completely absent from the DCS or if they are invalid. + + +.. _validate_generate_config: + +Configuration generation and validation +--------------------------------------- + +Patroni provides command-line interfaces for a Patroni :ref:`local configuration ` generation and validation. Using the ``patroni`` executable you can: + +- Create a sample local Patroni configuration; +- Create a Patroni configuration file for the locally running PostgreSQL instance (e.g. as a preparation step for the :ref:`Patroni integration `); +- Validate a given Patroni configuration file. + + +Sample Patroni configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code:: text + + patroni --generate-sample-config [configfile] + +Description +""""""""""" + +Generate a sample Patroni configuration file in ``yaml`` format. +Parameter values are defined using the :ref:`Environment configuration `, otherwise, if not set, the defaults used in Patroni or the ``#FIXME`` string for the values that should be later defined by the user. + +Some default values are defined based on the local setup: + + - **postgresql.listen**: the IP address returned by ``gethostname`` call for the current machine's hostname and the standard ``5432`` port. + - **postgresql.connect_address**: the IP address returned by ``gethostname`` call for the current machine's hostname and the standard ``5432`` port. + - **postgresql.authentication.rewind**: is only defined if the PostgreSQL version can be defined from the binary and the version is 11 or later. + - **restapi.listen**: IP address returned by ``gethostname`` call for the current machine's hostname and the standard ``8008`` port. + - **restapi.connect_address**: IP address returned by ``gethostname`` call for the current machine's hostname and the standard ``8008`` port. + +Parameters +"""""""""" + +``configfile`` - full path to the configuration file used to store the result. If not provided, the result is sent to ``stdout``. + + +Patroni configuration for a running instance +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code:: text + + patroni --generate-config [--dsn DSN] [configfile] + +Description +""""""""""" + +Generate a Patroni configuration in ``yaml`` format for the locally running PostgreSQL instance. +Either the provided DSN (takes precedence) or PostgreSQL `environment variables `__ will be used for the PostgreSQL connection. If the password is not provided, it should be entered via prompt. + +All the non-internal GUCs defined in the source Postgres instance, independently if they were set through a configuration file, through the postmaster command-line, or through environment variables, will be used as the source for the following Patroni configuration parameters: + + - **scope**: ``cluster_name`` GUC value; + - **postgresql.listen**: ``listen_addresses`` and ``port`` GUC values; + - **postgresql.datadir**: ``data_directory`` GUC value; + - **postgresql.parameters**: ``archive_command``, ``restore_command``, ``archive_cleanup_command``, ``recovery_end_command``, ``ssl_passphrase_command``, ``hba_file``, ``ident_file``, ``config_file`` GUC values; + - **bootstrap.dcs**: all other gathered PostgreSQL GUCs. + +If ``scope``, ``postgresql.listen`` or ``postgresql.datadir`` is not set from the Postgres GUCs, the respective :ref:`Environment configuration ` value is used. + +Other rules applied for the values definition: + + - **name**: ``PATRONI_NAME`` environment variable value if set, otherwise the current machine's hostname. + - **postgresql.bin_dir**: path to the Postgres binaries gathered from the running instance. + - **postgresql.connect_address**: the IP address returned by ``gethostname`` call for the current machine's hostname and the port used for the instance connection or the ``port`` GUC value. + - **postgresql.authentication.superuser**: the configuration used for the instance connection; + - **postgresql.pg_hba**: the lines gathered from the source instance's ``hba_file``. + - **postgresql.pg_ident**: the lines gathered from the source instance's ``ident_file``. + - **restapi.listen**: IP address returned by ``gethostname`` call for the current machine's hostname and the standard ``8008`` port. + - **restapi.connect_address**: IP address returned by ``gethostname`` call for the current machine's hostname and the standard ``8008`` port. + +Other parameters defined using :ref:`Environment configuration ` are also included into the configuration. + +Parameters +"""""""""" + +``configfile`` + Full path to the configuration file used to store the result. If not provided, result is sent to ``stdout``. + +``dsn`` + Optional DSN string for the local PostgreSQL instance to get GUC values from. + + +Validate Patroni configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code:: text + + patroni --validate-config [configfile] + +Description +""""""""""" + +Validate the given Patroni configuration and print the information about the failed checks. + +Parameters +"""""""""" + +``configfile`` + Full path to the configuration file to check. If not given or file does not exist, will try to read from the ``PATRONI_CONFIG_VARIABLE`` environment variable or, if not set, from the :ref:`Patroni environment variables `. From 3d527f5728fa8c6fadc55284cf854eead6a30a9b Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 25 Oct 2023 14:23:57 +0200 Subject: [PATCH 02/19] Improve formatting of generated config and validation of ints (#2928) - order sections similar to sample configs - add warnings and comments to `bootstrap.dcs` section. - add `tags` and `log` sections. - use discovered IPs in `postgresql.connect_address` and `postgresql.listen` - set `wal_level` to `replica` for PostgreSQL 9.6+ - make unit tests pass with python 3.6 - improve config validator so it doesn't complain when some ints are strings in YAML file. --- patroni/config_generator.py | 133 ++++++++++++++++++++++----------- patroni/validator.py | 59 ++++++++------- tests/test_config_generator.py | 60 ++++++++++----- 3 files changed, 161 insertions(+), 91 deletions(-) diff --git a/patroni/config_generator.py b/patroni/config_generator.py index 4bcb2819..c2b133c6 100644 --- a/patroni/config_generator.py +++ b/patroni/config_generator.py @@ -9,7 +9,7 @@ import yaml from getpass import getuser, getpass from contextlib import contextmanager -from typing import Any, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union +from typing import Any, Dict, Iterator, List, Optional, TextIO, Tuple, TYPE_CHECKING, Union if TYPE_CHECKING: # pragma: no cover from psycopg import Cursor from psycopg2 import cursor @@ -17,6 +17,7 @@ if TYPE_CHECKING: # pragma: no cover from . import psycopg from .config import Config from .exceptions import PatroniException +from .log import PatroniLogger from .postgresql.config import ConfigHandler, parse_dsn from .postgresql.misc import postgres_major_version_to_int from .utils import get_major_version, parse_bool, patch_config, read_stripped @@ -93,10 +94,20 @@ class AbstractConfigGenerator(abc.ABC): template_config: Dict[str, Any] = { 'scope': NO_VALUE_MSG, 'name': cls._HOSTNAME, + 'restapi': { + 'connect_address': cls._IP + ':8008', + 'listen': cls._IP + ':8008' + }, + 'log': { + 'level': PatroniLogger.DEFAULT_LEVEL, + 'traceback_level': PatroniLogger.DEFAULT_TRACEBACK_LEVEL, + 'format': PatroniLogger.DEFAULT_FORMAT, + 'max_queue_size': PatroniLogger.DEFAULT_MAX_QUEUE_SIZE + }, 'postgresql': { 'data_dir': NO_VALUE_MSG, - 'connect_address': NO_VALUE_MSG + ':5432', - 'listen': NO_VALUE_MSG + ':5432', + 'connect_address': cls._IP + ':5432', + 'listen': cls._IP + ':5432', 'bin_dir': '', 'authentication': { 'superuser': { @@ -109,9 +120,11 @@ class AbstractConfigGenerator(abc.ABC): } } }, - 'restapi': { - 'connect_address': cls._IP + ':8008', - 'listen': cls._IP + ':8008' + 'tags': { + 'failover_priority': 1, + 'noloadbalance': False, + 'clonefrom': True, + 'nosync': False, } } @@ -130,6 +143,72 @@ class AbstractConfigGenerator(abc.ABC): def generate(self) -> None: """Generate config and store in :attr:`~AbstractConfigGenerator.config`.""" + @staticmethod + def _format_block(block: Any, line_prefix: str = '') -> str: + """Format a single YAML block. + + .. note:: + Optionally the formatted block could be indented with the *line_prefix* + + :param block: the object that should be formatted to YAML. + :param line_prefix: is used for indentation. + + :returns: a formatted and indented *block*. + """ + return line_prefix + yaml.safe_dump(block, default_flow_style=False, line_break='\n', + allow_unicode=True, indent=2).strip().replace('\n', '\n' + line_prefix) + + def _format_config_section(self, section_name: str) -> Iterator[str]: + """Format and yield as single section of the current :attr:`~AbstractConfigGenerator.config`. + + .. note:: + If the section is a :class:`dict` object we put an empty line before it. + + :param section_name: a section name in the :attr:`~AbstractConfigGenerator.config`. + + :yields: a formatted section in case if it exists in the :attr:`~AbstractConfigGenerator.config`. + """ + if section_name in self.config: + if isinstance(self.config[section_name], dict): + yield '' + yield self._format_block({section_name: self.config[section_name]}) + + def _format_config(self) -> Iterator[str]: + """Format current :attr:`~AbstractConfigGenerator.config` and enrich it with some comments. + + :yields: formatted lines or blocks that represent a text output of the YAML document. + """ + for name in ('scope', 'namespace', 'name', 'log', 'restapi', 'ctl' 'citus', + 'consul', 'etcd', 'etcd3', 'exhibitor', 'kubernetes', 'raft', 'zookeeper'): + yield from self._format_config_section(name) + + if 'bootstrap' in self.config: + yield '\n# The bootstrap configuration. Works only when the cluster is not yet initialized.' + yield '# If the cluster is already initialized, all changes in the `bootstrap` section are ignored!' + yield 'bootstrap:' + if 'dcs' in self.config['bootstrap']: + yield ' # This section will be written into :///config after initializing' + yield ' # new cluster and all other cluster members will use it as a `global configuration`.' + yield ' # WARNING! If you want to change any of the parameters that were set up' + yield ' # via `bootstrap.dcs` section, please use `patronictl edit-config`!' + yield ' dcs:' + for name in ('loop_wait', 'retry_timeout', 'ttl'): + if name in self.config['bootstrap']['dcs']: + yield self._format_block({name: self.config['bootstrap']['dcs'].pop(name)}, ' ') + + for name, value in self.config['bootstrap']['dcs'].items(): + yield self._format_block({name: value}, ' ') + + for name in ('postgresql', 'watchdog', 'tags'): + yield from self._format_config_section(name) + + def _write_config_to_fd(self, fd: TextIO) -> None: + """Format and write current :attr:`~AbstractConfigGenerator.config` to provided file descriptor. + + :param fd: where to write the config file. Could be ``sys.stdout`` or the real file. + """ + fd.write('\n'.join(self._format_config())) + def write_config(self) -> None: """Write current :attr:`~AbstractConfigGenerator.config` to the output file if provided, to stdout otherwise.""" if self.output_file: @@ -137,9 +216,9 @@ class AbstractConfigGenerator(abc.ABC): if dir_path and not os.path.isdir(dir_path): os.makedirs(dir_path) with open(self.output_file, 'w', encoding='UTF-8') as output_file: - yaml.safe_dump(self.config, output_file, default_flow_style=False, allow_unicode=True) + self._write_config_to_fd(output_file) else: - yaml.safe_dump(self.config, sys.stdout, default_flow_style=False, allow_unicode=True) + self._write_config_to_fd(sys.stdout) class SampleConfigGenerator(AbstractConfigGenerator): @@ -182,6 +261,9 @@ class SampleConfigGenerator(AbstractConfigGenerator): self.config['bootstrap']['dcs']['postgresql']['parameters'][wal_keep_param] = \ ConfigHandler.CMDLINE_OPTIONS[wal_keep_param][0] + wal_level = 'hot_standby' if self.pg_major < 90600 else 'replica' + self.config['bootstrap']['dcs']['postgresql']['parameters']['wal_level'] = wal_level + self.config['bootstrap']['dcs']['postgresql']['use_pg_rewind'] = True if self.pg_major >= 110000: self.config['postgresql']['authentication'].setdefault( @@ -411,41 +493,6 @@ class RunningClusterConfigGenerator(AbstractConfigGenerator): def generate_config(output_file: str, sample: bool, dsn: Optional[str]) -> None: """Generate Patroni configuration file. - Gather all the available non-internal GUC values having configuration file, postmaster command line or environment - variable as a source and store them in the appropriate part of Patroni configuration (``postgresql.parameters`` or - ``bootstrap.dcs.postgresql.parameters``). Either the provided DSN (takes precedence) or PG ENV vars will be used - for the connection. If password is not provided, it should be entered via prompt. - - The created configuration contains: - * ``scope``: ``cluster_name`` GUC value or ``PATRONI_SCOPE ENV`` variable value if available. - * ``name``: ``PATRONI_NAME`` ENV variable value if set, otherwise hostname. - - * ``bootstrap.dcs``: section with all the parameters (incl. the majority of PG GUCs) set to their default values - defined by Patroni and adjusted by the source instances's configuration values. - - * ``postgresql.parameters``: the source instance's ``archive_command``, ``restore_command``, - ``archive_cleanup_command``, ``recovery_end_command``, ``ssl_passphrase_command``, ``hba_file``, ``ident_file``, - ``config_file`` GUC values. - - * ``postgresql.bin_dir``: path to Postgres binaries gathered from the running instance or, if not available, - the value of ``PATRONI_POSTGRESQL_BIN_DIR`` ENV variable. Otherwise, an empty string. - - * ``postgresql.datadir``: the value gathered from the corresponding PG GUC. - * ``postgresql.listen``: source instance's ``listen_addresses`` and port GUC values. - * ``postgresql.connect_address``: if possible, generated from the connection params. - * ``postgresql.authentication``: - - * superuser and replication users defined (if possible, usernames are set from the respective Patroni ENV vars, - otherwise the default ``postgres`` and ``replicator`` values are used). - If not a sample config, either DSN or PG ENV vars are used to define superuser authentication parameters. - - * rewind user is defined only for sample config, if PG version can be defined and PG version is >=11 - (if possible, username is set from the respective Patroni ENV var). - - * ``bootstrap.dcs.postgresql.use_pg_rewind`` set to ``True`` for a sample config only. - * ``postgresql.pg_hba`` defaults or the lines gathered from the source instance's ``hba_file``. - * ``postgresql.pg_ident`` the lines gathered from the source instance's ``ident_file``. - :param output_file: Full path to the configuration file to be used. If not provided, result is sent to ``stdout``. :param sample: Optional flag. If set, no source instance will be used - generate config with some sane defaults. :param dsn: Optional DSN string for the local instance to get GUC values from. diff --git a/patroni/validator.py b/patroni/validator.py index 122b795e..10bc7b2a 100644 --- a/patroni/validator.py +++ b/patroni/validator.py @@ -814,27 +814,28 @@ def assert_(condition: bool, message: str = "Wrong value") -> None: class IntValidator(object): """Validate an integer setting. - :cvar expected_type: the expected Python type for an integer setting (:class:`int`). :ivar min: minimum allowed value for the setting, if any. :ivar max: maximum allowed value for the setting, if any. :ivar base_unit: the base unit to convert the value to before checking if it's within *min* and *max* range. + :ivar expected_type: the expected Python type. :ivar raise_assert: if an ``assert`` test should be performed regarding expected type and valid range. """ - expected_type = int - def __init__(self, min: OptionalType[int] = None, max: OptionalType[int] = None, - base_unit: OptionalType[str] = None, raise_assert: bool = False) -> None: + base_unit: OptionalType[str] = None, expected_type: Any = None, raise_assert: bool = False) -> None: """Create an :class:`IntValidator` object with the given rules. :param min: minimum allowed value for the setting, if any. :param max: maximum allowed value for the setting, if any. :param base_unit: the base unit to convert the value to before checking if it's within *min* and *max* range. + :param expected_type: the expected Python type. :param raise_assert: if an ``assert`` test should be performed regarding expected type and valid range. """ self.min = min self.max = max self.base_unit = base_unit + if expected_type: + self.expected_type = expected_type self.raise_assert = raise_assert def __call__(self, value: Any) -> bool: @@ -953,36 +954,36 @@ schema = Schema({ Optional("allowlist_include_members"): bool, Optional("http_extra_headers"): dict, Optional("https_extra_headers"): dict, - Optional("request_queue_size"): IntValidator(min=0, max=4096, raise_assert=True) + Optional("request_queue_size"): IntValidator(min=0, max=4096, expected_type=int, raise_assert=True) }, Optional("bootstrap"): { "dcs": { - Optional("ttl"): int, - Optional("loop_wait"): int, - Optional("retry_timeout"): int, - Optional("maximum_lag_on_failover"): int, - Optional("maximum_lag_on_syncnode"): int, + Optional("ttl"): IntValidator(min=20, raise_assert=True), + Optional("loop_wait"): IntValidator(min=1, raise_assert=True), + Optional("retry_timeout"): IntValidator(min=3, raise_assert=True), + Optional("maximum_lag_on_failover"): IntValidator(min=0, raise_assert=True), + Optional("maximum_lag_on_syncnode"): IntValidator(min=-1, raise_assert=True), Optional("postgresql"): { Optional("parameters"): { - Optional("max_connections"): int, - Optional("max_locks_per_transaction"): int, - Optional("max_prepared_transactions"): int, - Optional("max_replication_slots"): int, - Optional("max_wal_senders"): int, - Optional("max_worker_processes"): int + Optional("max_connections"): IntValidator(1, 262143, raise_assert=True), + Optional("max_locks_per_transaction"): IntValidator(10, 2147483647, raise_assert=True), + Optional("max_prepared_transactions"): IntValidator(0, 262143, raise_assert=True), + Optional("max_replication_slots"): IntValidator(0, 262143, raise_assert=True), + Optional("max_wal_senders"): IntValidator(0, 262143, raise_assert=True), + Optional("max_worker_processes"): IntValidator(0, 262143, raise_assert=True), }, Optional("use_pg_rewind"): bool, Optional("pg_hba"): [str], Optional("pg_ident"): [str], - Optional("pg_ctl_timeout"): int, + Optional("pg_ctl_timeout"): IntValidator(min=0, raise_assert=True), Optional("use_slots"): bool, }, - Optional("primary_start_timeout"): int, - Optional("primary_stop_timeout"): int, + Optional("primary_start_timeout"): IntValidator(min=0, raise_assert=True), + Optional("primary_stop_timeout"): IntValidator(min=0, raise_assert=True), Optional("standby_cluster"): { Or("host", "port", "restore_command"): Case({ "host": str, - "port": int, + "port": IntValidator(max=65535, expected_type=int, raise_assert=True), "restore_command": str }), Optional("primary_slot_name"): str, @@ -992,7 +993,7 @@ schema = Schema({ }, Optional("synchronous_mode"): bool, Optional("synchronous_mode_strict"): bool, - Optional("synchronous_node_count"): int + Optional("synchronous_node_count"): IntValidator(min=1, raise_assert=True), }, Optional("initdb"): [Or(str, dict)], Optional("method"): str @@ -1003,7 +1004,7 @@ schema = Schema({ "host": validate_host_port, "url": str }), - Optional("port"): int, + Optional("port"): IntValidator(max=65535, expected_type=int, raise_assert=True), Optional("scheme"): str, Optional("token"): str, Optional("verify"): bool, @@ -1023,8 +1024,8 @@ schema = Schema({ "etcd3": validate_etcd, "exhibitor": { "hosts": [str], - "port": IntValidator(max=65535, raise_assert=True), - Optional("pool_interval"): int + "port": IntValidator(max=65535, expected_type=int, raise_assert=True), + Optional("poll_interval"): IntValidator(min=1, expected_type=int, raise_assert=True), }, "raft": { "self_addr": validate_connect_address, @@ -1055,14 +1056,14 @@ schema = Schema({ Optional("tmp_role_label"): str, Optional("use_endpoints"): bool, Optional("pod_ip"): Or(is_ipv4_address, is_ipv6_address), - Optional("ports"): [{"name": str, "port": int}], + Optional("ports"): [{"name": str, "port": IntValidator(max=65535, expected_type=int, raise_assert=True)}], Optional("cacert"): str, Optional("retriable_http_codes"): Or(int, [int]), }, }), Optional("citus"): { "database": str, - "group": int + "group": IntValidator(min=0, expected_type=int, raise_assert=True), }, "postgresql": { "listen": validate_host_port_listen_multiple_hosts, @@ -1089,18 +1090,18 @@ schema = Schema({ }, Optional("pg_hba"): [str], Optional("pg_ident"): [str], - Optional("pg_ctl_timeout"): int, + Optional("pg_ctl_timeout"): IntValidator(min=0, raise_assert=True), Optional("use_pg_rewind"): bool }, Optional("watchdog"): { Optional("mode"): validate_watchdog_mode, Optional("device"): str, - Optional("safety_margin"): int + Optional("safety_margin"): IntValidator(min=-1, expected_type=int, raise_assert=True), }, Optional("tags"): { AtMostOne("nofailover", "failover_priority"): Case({ "nofailover": bool, - "failover_priority": IntValidator(min=0, raise_assert=True), + "failover_priority": IntValidator(min=0, expected_type=int, raise_assert=True), }), Optional("clonefrom"): bool, Optional("noloadbalance"): bool, diff --git a/tests/test_config_generator.py b/tests/test_config_generator.py index e702f957..1b436bb0 100644 --- a/tests/test_config_generator.py +++ b/tests/test_config_generator.py @@ -1,14 +1,16 @@ import os import psutil import unittest +import yaml from . import MockConnect, MockCursor, MockConnectionInfo from copy import deepcopy -from mock import MagicMock, Mock, PropertyMock, mock_open, patch +from mock import MagicMock, Mock, PropertyMock, mock_open as _mock_open, patch from patroni.__main__ import main as _main from patroni.config import Config from patroni.config_generator import AbstractConfigGenerator, get_address, NO_VALUE_MSG +from patroni.log import PatroniLogger from patroni.utils import patch_config from . import psycopg_connect @@ -17,6 +19,14 @@ HOSTNAME = 'test_hostname' IP = '1.9.8.4' +def mock_open(*args, **kwargs): + ret = _mock_open(*args, **kwargs) + ret.return_value.__iter__ = lambda o: iter(o.readline, '') + if not kwargs.get('read_data'): + ret.return_value.readline = Mock(return_value=None) + return ret + + @patch('patroni.psycopg.connect', psycopg_connect) @patch('builtins.open', MagicMock()) @patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 16.2")) @@ -51,13 +61,23 @@ class TestGenerateConfig(unittest.TestCase): self.config = { 'scope': self.environ['PATRONI_SCOPE'], 'name': HOSTNAME, + 'log': { + 'level': PatroniLogger.DEFAULT_LEVEL, + 'traceback_level': PatroniLogger.DEFAULT_TRACEBACK_LEVEL, + 'format': PatroniLogger.DEFAULT_FORMAT, + 'max_queue_size': PatroniLogger.DEFAULT_MAX_QUEUE_SIZE + }, + 'restapi': { + 'connect_address': self.environ['PATRONI_RESTAPI_CONNECT_ADDRESS'], + 'listen': self.environ['PATRONI_RESTAPI_LISTEN'] + }, 'bootstrap': { 'dcs': dynamic_config }, 'postgresql': { - 'connect_address': NO_VALUE_MSG + ':5432', + 'connect_address': IP + ':5432', 'data_dir': NO_VALUE_MSG, - 'listen': NO_VALUE_MSG + ':5432', + 'listen': IP + ':5432', 'pg_hba': ['host all all all md5', f'host replication {self.environ["PATRONI_REPLICATION_USERNAME"]} all md5'], 'authentication': {'superuser': {'username': self.environ['PATRONI_SUPERUSER_USERNAME'], @@ -68,10 +88,6 @@ class TestGenerateConfig(unittest.TestCase): 'bin_dir': self.environ['PATRONI_POSTGRESQL_BIN_DIR'], 'bin_name': {'postgres': self.environ['PATRONI_POSTGRESQL_BIN_POSTGRES']}, 'parameters': {'password_encryption': 'md5'} - }, - 'restapi': { - 'connect_address': self.environ['PATRONI_RESTAPI_CONNECT_ADDRESS'], - 'listen': self.environ['PATRONI_RESTAPI_LISTEN'] } } @@ -119,6 +135,12 @@ class TestGenerateConfig(unittest.TestCase): }, 'rewind': None }, + }, + 'tags': { + 'failover_priority': 1, + 'noloadbalance': False, + 'clonefrom': True, + 'nosync': False, } } patch_config(self.config, conf) @@ -139,22 +161,21 @@ class TestGenerateConfig(unittest.TestCase): ] @patch('os.makedirs') - @patch('yaml.safe_dump') - def test_generate_sample_config_pre_13_dir_creation(self, mock_config_dump, mock_makedir): + def test_generate_sample_config_pre_13_dir_creation(self, mock_makedir): with patch('sys.argv', ['patroni.py', '--generate-sample-config', '/foo/bar.yml']), \ patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 9.4.3")) as pg_bin_mock, \ + patch('builtins.open', _mock_open()) as mocked_file, \ self.assertRaises(SystemExit) as e: _main() + self.assertEqual(self.config, yaml.safe_load(mocked_file().write.call_args_list[0][0][0])) self.assertEqual(e.exception.code, 0) - self.assertEqual(self.config, mock_config_dump.call_args[0][0]) mock_makedir.assert_called_once() pg_bin_mock.assert_called_once_with([os.path.join(self.environ['PATRONI_POSTGRESQL_BIN_DIR'], self.environ['PATRONI_POSTGRESQL_BIN_POSTGRES']), '--version']) @patch('os.makedirs', Mock()) - @patch('yaml.safe_dump') - def test_generate_sample_config_16(self, mock_config_dump): + def test_generate_sample_config_16(self): conf = { 'bootstrap': { 'dcs': { @@ -182,14 +203,15 @@ class TestGenerateConfig(unittest.TestCase): patch_config(self.config, conf) with patch('sys.argv', ['patroni.py', '--generate-sample-config', '/foo/bar.yml']), \ + patch('builtins.open', _mock_open()) as mocked_file, \ self.assertRaises(SystemExit) as e: _main() + self.assertEqual(self.config, yaml.safe_load(mocked_file().write.call_args_list[0][0][0])) self.assertEqual(e.exception.code, 0) - self.assertEqual(self.config, mock_config_dump.call_args[0][0]) @patch('os.makedirs', Mock()) - @patch('yaml.safe_dump') - def test_generate_config_running_instance_16(self, mock_config_dump): + @patch('sys.stdout') + def test_generate_config_running_instance_16(self, mock_sys_stdout): self._set_running_instance_config_vals() with patch('builtins.open', Mock(side_effect=self._get_running_instance_open_res())), \ @@ -198,11 +220,11 @@ class TestGenerateConfig(unittest.TestCase): self.assertRaises(SystemExit) as e: _main() self.assertEqual(e.exception.code, 0) - self.assertEqual(self.config, mock_config_dump.call_args[0][0]) + self.assertEqual(self.config, yaml.safe_load(mock_sys_stdout.write.call_args_list[0][0][0])) @patch('os.makedirs', Mock()) - @patch('yaml.safe_dump') - def test_generate_config_running_instance_16_connect_from_env(self, mock_config_dump): + @patch('sys.stdout') + def test_generate_config_running_instance_16_connect_from_env(self, mock_sys_stdout): self._set_running_instance_config_vals() # su auth params and connect host from env os.environ['PGCHANNELBINDING'] = \ @@ -245,7 +267,7 @@ class TestGenerateConfig(unittest.TestCase): self.assertRaises(SystemExit) as e: _main() self.assertEqual(e.exception.code, 0) - self.assertEqual(self.config, mock_config_dump.call_args[0][0]) + self.assertEqual(self.config, yaml.safe_load(mock_sys_stdout.write.call_args_list[0][0][0])) def test_generate_config_running_instance_errors(self): # 1. Wrong DSN format From bb90feb393aa8dfaf24c28acaf4519137ca7822e Mon Sep 17 00:00:00 2001 From: Israel Date: Wed, 25 Oct 2023 10:01:08 -0300 Subject: [PATCH 03/19] Add support for additional parameters on custom bootstrap (#2927) Previous to this commit, if a user would ever like to add parameters to the custom bootstrap script call, they would need to configure Patroni like this: ``` bootstrap: method: custom_method_name custom_method_name: command: /path/to/my/custom_script --arg1=value1 --arg2=value2 ... ``` This commit extends that so we achieve a similar behavior that is seen when using `create_replica_methods`, i.e., we also allow the following syntax: ``` bootstrap: method: custom_method_name custom_method_name: command: /path/to/my/custom_script arg1: value1 arg2: value2 ``` All keys in the mapping which are not recognized by Patroni, will be dealt with as if they were additional named arguments to be passed down to the `command` call. References: PAT-218. --- docs/replica_bootstrap.rst | 25 ++++++++++++++++---- features/backup_restore.py | 1 + features/environment.py | 22 ++++++++++-------- features/steps/standby_cluster.py | 4 +--- patroni/postgresql/bootstrap.py | 38 +++++++++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 18 deletions(-) diff --git a/docs/replica_bootstrap.rst b/docs/replica_bootstrap.rst index 5ae53103..861a0e86 100644 --- a/docs/replica_bootstrap.rst +++ b/docs/replica_bootstrap.rst @@ -43,16 +43,31 @@ in the configuration files, Patroni supplies two cluster-specific ones: Passing these two additional flags can be disabled by setting a special ``no_params`` parameter to ``True``. -If the bootstrap script returns 0, Patroni tries to configure and start the PostgreSQL instance produced by it. If any +If the bootstrap script returns ``0``, Patroni tries to configure and start the PostgreSQL instance produced by it. If any of the intermediate steps fail, or the script returns a non-zero value, Patroni assumes that the bootstrap has failed, cleans up after itself and releases the initialize lock to give another node the opportunity to bootstrap. If a ``recovery_conf`` block is defined in the same section as the custom bootstrap method, Patroni will generate a -``recovery.conf`` before starting the newly bootstrapped instance. Typically, such recovery.conf should contain at least -one of the ``recovery_target_*`` parameters, together with the ``recovery_target_timeline`` set to ``promote``. +``recovery.conf`` before starting the newly bootstrapped instance (or set the recovery settings on Postgres configuration if +running PostgreSQL >= 12). +Typically, such recovery configuration should contain at least one of the ``recovery_target_*`` parameters, together with the ``recovery_target_timeline`` set to ``promote``. -If ``keep_existing_recovery_conf`` is defined and set to ``True``, Patroni will not remove the existing ``recovery.conf`` file if it exists. -This is useful when bootstrapping from a backup with tools like pgBackRest that generate the appropriate ``recovery.conf`` for you. +If ``keep_existing_recovery_conf`` is defined and set to ``True``, Patroni will not remove the existing ``recovery.conf`` file if it exists (PostgreSQL <= 11). +Similarly, in that case Patroni will not remove the existing ``recovery.signal`` or ``standby.signal`` if either exists, nor will it override the configured recovery settings (PostgreSQL >= 12). +This is useful when bootstrapping from a backup with tools like pgBackRest that generate the appropriate recovery configuration for you. + +Besides that, any additional key/value pairs informed in the custom bootstrap method configuration will be passed as arguments to ``command`` in the format ``--name=value``. For example: + +.. code:: YAML + + bootstrap: + method: + : + command: + arg1: value1 + arg2: value2 + +Makes the configured ``command`` to be called additionally with ``--arg1=value1 --arg2=value2`` command-line arguments. .. note:: Bootstrap methods are neither chained, nor fallen-back to the default one in case the primary one fails diff --git a/features/backup_restore.py b/features/backup_restore.py index 5aa973c4..9b926b54 100755 --- a/features/backup_restore.py +++ b/features/backup_restore.py @@ -6,6 +6,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--datadir", required=True) parser.add_argument("--sourcedir", required=True) + parser.add_argument("--test-argument", required=True) args, _ = parser.parse_known_args() shutil.copytree(args.sourcedir, args.datadir) diff --git a/features/environment.py b/features/environment.py index 1c3e654b..a0367657 100644 --- a/features/environment.py +++ b/features/environment.py @@ -892,22 +892,28 @@ class PatroniPoolController(object): } self.start(to_name, custom_config=custom_config) + def backup_restore_config(self, params=None): + return { + 'command': (self.BACKUP_RESTORE_SCRIPT + + ' --sourcedir=' + os.path.join(self.patroni_path, 'data', 'basebackup')).replace('\\', '/'), + 'test-argument': 'test-value', # test config mapping approach on custom bootstrap/replica creation + **(params or {}), + } + def bootstrap_from_backup(self, name, cluster_name): custom_config = { 'scope': cluster_name, 'bootstrap': { 'method': 'backup_restore', - 'backup_restore': { - 'command': (self.BACKUP_RESTORE_SCRIPT + ' --sourcedir=' - + os.path.join(self.patroni_path, 'data', 'basebackup').replace('\\', '/')), + 'backup_restore': self.backup_restore_config({ 'recovery_conf': { 'recovery_target_action': 'promote', 'recovery_target_timeline': 'latest', 'restore_command': (self.ARCHIVE_RESTORE_SCRIPT + ' --mode restore ' + '--dirname {} --filename %f --pathname %p').format( os.path.join(self.patroni_path, 'data', 'wal_archive_clone').replace('\\', '/')) - } - } + }, + }) }, 'postgresql': { 'authentication': { @@ -928,11 +934,7 @@ class PatroniPoolController(object): .format(os.path.join(self.patroni_path, 'data', 'wal_archive').replace('\\', '/')) }, 'create_replica_methods': ['no_leader_bootstrap'], - 'no_leader_bootstrap': { - 'command': (self.BACKUP_RESTORE_SCRIPT + ' --sourcedir=' - + os.path.join(self.patroni_path, 'data', 'basebackup').replace('\\', '/')), - 'no_leader': '1' - } + 'no_leader_bootstrap': self.backup_restore_config({'no_leader': '1'}) } } self.start(name, custom_config=custom_config) diff --git a/features/steps/standby_cluster.py b/features/steps/standby_cluster.py index a428130c..1bf3b3fb 100644 --- a/features/steps/standby_cluster.py +++ b/features/steps/standby_cluster.py @@ -15,9 +15,7 @@ def start_patroni(context, name, cluster_name): "scope": cluster_name, "postgresql": { "callbacks": callbacks(context, name), - "backup_restore": { - "command": (context.pctl.PYTHON + " features/backup_restore.py --sourcedir=" - + os.path.join(context.pctl.patroni_path, 'data', 'basebackup').replace('\\', '/'))} + "backup_restore": context.pctl.backup_restore_config() } }) diff --git a/patroni/postgresql/bootstrap.py b/patroni/postgresql/bootstrap.py index 8a9b0fba..751c0797 100644 --- a/patroni/postgresql/bootstrap.py +++ b/patroni/postgresql/bootstrap.py @@ -151,9 +151,47 @@ class Bootstrap(object): os.unlink(trigger_file) def _custom_bootstrap(self, config: Any) -> bool: + """Bootstrap a fresh Patroni cluster using a custom method provided by the user. + + :param config: configuration used for running a custom bootstrap method. It comes from the Patroni YAML file, + so it is expected to be a :class:`dict`. + + .. note:: + *config* must contain a ``command`` key, which value is the command or script to perform the custom + bootstrap procedure. The exit code of the ``command`` dictates if the bootstrap succeeded or failed. + + When calling ``command``, Patroni will pass the following arguments to the ``command`` call: + + * ``--scope``: contains the value of ``scope`` configuration; + * ``--data_dir``: contains the value of the ``postgresql.data_dir`` configuration. + + You can avoid that behavior by filling the optional key ``no_params`` with the value ``False`` in the + configuration file, which will instruct Patroni to not pass these parameters to the ``command`` call. + + Besides that, a couple more keys are supported in *config*, but optional: + + * ``keep_existing_recovery_conf``: if ``True``, instruct Patroni to not remove the existing + ``recovery.conf`` (PostgreSQL <= 11), to not discard recovery parameters from the configuration + (PostgreSQL >= 12), and to not remove the files ``recovery.signal`` or ``standby.signal`` + (PostgreSQL >= 12). This is specially useful when you are restoring backups through tools like + pgBackRest and Barman, in which case they generated the appropriate recovery settings for you; + * ``recovery_conf``: a section containing a map, where each key is the name of a recovery related + setting, and the value is the value of the corresponding setting. + + Any key/value other than the ones that were described above will be interpreted as additional arguments for + the ``command`` call. They will all be added to the call in the format ``--key=value``. + + :returns: ``True`` if the bootstrap was successful, i.e. the execution of the custom ``command`` from *config* + exited with code ``0``, ``False`` otherwise. + """ self._postgresql.set_state('running custom bootstrap script') params = [] if config.get('no_params') else ['--scope=' + self._postgresql.scope, '--datadir=' + self._postgresql.data_dir] + # Add custom parameters specified by the user + reserved_args = {'no_params', 'keep_existing_recovery_conf', 'recovery_conf', 'scope', 'datadir'} + for arg, val in config.items(): + if arg not in reserved_args: + params.append(f"--{arg}={val}") try: logger.info('Running custom bootstrap script: %s', config['command']) if self._postgresql.cancellable.call(shlex.split(config['command']) + params) != 0: From ce10e5fccc8eb6fc3e0e6bd177f6958d463904fd Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 25 Oct 2023 16:13:30 +0200 Subject: [PATCH 04/19] Release v3.2.0 (#2930) - bump version - bump pyright and apply fixes - update release notes --- .github/workflows/tests.yaml | 2 +- docs/releases.rst | 72 ++++++++++++++++++++++++++++++++++++ patroni/api.py | 2 +- patroni/version.py | 2 +- 4 files changed, 75 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index aba3e751..81119f25 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -174,7 +174,7 @@ jobs: - uses: jakebailey/pyright-action@v1 with: - version: 1.1.326 + version: 1.1.333 docs: runs-on: ubuntu-latest diff --git a/docs/releases.rst b/docs/releases.rst index 34c44047..cd2714b6 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -3,6 +3,78 @@ Release notes ============= +Version 3.2.0 +------------- + +**Deprecation notice** + +- The ``bootstrap.users`` support will be removed in version 4.0.0. If you need to create users after deploying a new cluster please use the ``bootstrap.post_bootstrap`` hook for that. + + +**Breaking changes** + +- Enforce ``loop_wait + 2*retry_timeout <= ttl`` rule and hard-code minimal possible values (Alexander Kukushkin) + + Minimal values: ``loop_wait=2``, ``retry_timeout=3``, ``ttl=20``. In case values are smaller or violate the rule they are adjusted and a warning is written to Patroni logs. + + +**New features** + +- Failover priority (Mark Pekala) + + With the help of ``tags.failover_priority`` it's now possible to make a node more preferred during the leader race. More details in the documentation (ref tags). + +- Implemented ``patroni --generate-config [--dsn DSN]`` and ``patroni --generate-sample-config`` (Polina Bungina) + + It allows to generate a config file for the running PostgreSQL cluster or a sample config file for the new Patroni cluster. + +- Use a dedicated connection to Postgres for Patroni REST API (Alexander Kukushkin) + + It helps to avoid blocking the main heartbeat loop if the system is under stress. + +- Enrich some endpoints with the ``name`` of the node (sskserk) + + For the monitoring endpoint ``name`` is added next to the ``scope`` and for metrics endpoint the ``name`` is added to tags. + +- Ensure strict failover/switchover difference (Polina Bungina) + + Be more precise in log messages and allow failing over to an asynchronous node in a healthy synchronous cluster. + +- Make permanent physical replication slots behave similarly to permanent logical slots (Alexander Kukushkin) + + Create permanent physical replication slots on all nodes that are allowed to become the leader and use ``pg_replication_slot_advance()`` function to advance ``restart_lsn`` for slots on standby nodes. + +- Add capability of specifying namespace through ``--dcs`` argument in ``patronictl`` (Israel Barth Rubio) + + It could be handy if ``patronictl`` is used without a configuration file. + +- Add support for additional parameters in custom bootstrap configuration (Israel Barth Rubio) + + Previously it was only possible to add custom arguments to the ``command`` and now one could list them as a mapping. + + +**Improvements** + +- Set ``citus.local_hostname`` GUC to the same value which is used by Patroni to connect to the Postgres (Alexander Kukushkin) + + There are cases when Citus wants to have a connection to the local Postgres. By default it uses ``localhost``, which is not always available. + + +**Bugfixes** + +- Ignore ``synchronous_mode`` setting in a standby cluster (Polina Bungina) + + Postgres doesn't support cascading synchronous replication and not ignoring ``synchronous_mode`` was breaking a switchover in a standby cluster. + +- Handle SIGCHLD for ``on_reload`` callback (Alexander Kukushkin) + + Not doing so results in a zombie process, which is reaped only when the next ``on_reload`` is executed. + +- Handle ``AuthOldRevision`` error when working with Etcd v3 (Alexander Kukushkin, Kenny Do) + + The error is raised if Etcd is configured to use JWT and when the user database in Etcd is updated. + + Version 3.1.2 ------------- diff --git a/patroni/api.py b/patroni/api.py index e1b71337..6a89b1ba 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -103,7 +103,7 @@ class RestApiHandler(BaseHTTPRequestHandler): if TYPE_CHECKING: # pragma: no cover assert isinstance(server, RestApiServer) super(RestApiHandler, self).__init__(request, client_address, server) - self.server: 'RestApiServer' = server + self.server: 'RestApiServer' = server # pyright: ignore [reportIncompatibleVariableOverride] self.__start_time: float = 0.0 self.path_query: Dict[str, List[str]] = {} diff --git a/patroni/version.py b/patroni/version.py index b2ab33ce..e5bcac2d 100644 --- a/patroni/version.py +++ b/patroni/version.py @@ -2,4 +2,4 @@ :var __version__: the current Patroni version. """ -__version__ = '3.1.2' +__version__ = '3.2.0' From c3dce4683085185ba5bcf0a20b53404466ebccb4 Mon Sep 17 00:00:00 2001 From: Aras Mumcuyan Date: Mon, 30 Oct 2023 11:46:36 +0100 Subject: [PATCH 05/19] Add ability to pass auth_data to zk client (#2932) --- docs/ENVIRONMENT.rst | 1 + docs/yaml_configuration.rst | 1 + patroni/config.py | 4 ++-- patroni/dcs/zookeeper.py | 3 ++- patroni/validator.py | 3 ++- 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst index bd928af9..ad7c46b8 100644 --- a/docs/ENVIRONMENT.rst +++ b/docs/ENVIRONMENT.rst @@ -85,6 +85,7 @@ ZooKeeper - **PATRONI\_ZOOKEEPER\_KEY\_PASSWORD**: (optional) The client key password. - **PATRONI\_ZOOKEEPER\_VERIFY**: (optional) Whether to verify certificate or not. Defaults to ``true``. - **PATRONI\_ZOOKEEPER\_SET\_ACLS**: (optional) If set, configure Kazoo to apply a default ACL to each ZNode that it creates. ACLs will assume 'x509' schema and should be specified as a dictionary with the principal as the key and one or more permissions as a list in the value. Permissions may be one of ``CREATE``, ``READ``, ``WRITE``, ``DELETE`` or ``ADMIN``. For example, ``set_acls: {CN=principal1: [CREATE, READ], CN=principal2: [ALL]}``. +- **PATRONI\_ZOOKEEPER\_AUTH\_DATA**: (optional) Authentication credentials to use for the connection. Should be a dictionary in the form that `scheme` is the key and `credential` is the value. Defaults to empty dictionary. .. note:: It is required to install ``kazoo>=2.6.0`` to support SSL. diff --git a/docs/yaml_configuration.rst b/docs/yaml_configuration.rst index fc313fd0..a4a33134 100644 --- a/docs/yaml_configuration.rst +++ b/docs/yaml_configuration.rst @@ -133,6 +133,7 @@ ZooKeeper - **key_password**: (optional) The client key password. - **verify**: (optional) Whether to verify certificate or not. Defaults to ``true``. - **set_acls**: (optional) If set, configure Kazoo to apply a default ACL to each ZNode that it creates. ACLs will assume 'x509' schema and should be specified as a dictionary with the principal as the key and one or more permissions as a list in the value. Permissions may be one of ``CREATE``, ``READ``, ``WRITE``, ``DELETE`` or ``ADMIN``. For example, ``set_acls: {CN=principal1: [CREATE, READ], CN=principal2: [ALL]}``. +- **auth_data**: (optional) Authentication credentials to use for the connection. Should be a dictionary in the form that `scheme` is the key and `credential` is the value. Defaults to empty dictionary. .. note:: It is required to install ``kazoo>=2.6.0`` to support SSL. diff --git a/patroni/config.py b/patroni/config.py index ee0b2710..00dba9d9 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -791,7 +791,7 @@ class Config(object): 'SERVICE_TAGS', 'NAMESPACE', 'CONTEXT', 'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL', 'POD_IP', 'PORTS', 'LABELS', 'BYPASS_API_SERVICE', 'RETRIABLE_HTTP_CODES', 'KEY_PASSWORD', 'USE_SSL', 'SET_ACLS', 'GROUP', 'DATABASE', 'LEADER_LABEL_VALUE', 'FOLLOWER_LABEL_VALUE', - 'STANDBY_LEADER_LABEL_VALUE', 'TMP_ROLE_LABEL') and name: + 'STANDBY_LEADER_LABEL_VALUE', 'TMP_ROLE_LABEL', 'AUTH_DATA') and name: value = os.environ.pop(param) if name == 'CITUS': if suffix == 'GROUP': @@ -802,7 +802,7 @@ class Config(object): value = value and parse_int(value) elif suffix in ('HOSTS', 'PORTS', 'CHECKS', 'SERVICE_TAGS', 'RETRIABLE_HTTP_CODES'): value = value and _parse_list(value) - elif suffix in ('LABELS', 'SET_ACLS'): + elif suffix in ('LABELS', 'SET_ACLS', 'AUTH_DATA'): value = _parse_dict(value) elif suffix in ('USE_PROXIES', 'REGISTER_SERVICE', 'USE_ENDPOINTS', 'BYPASS_API_SERVICE', 'VERIFY'): value = parse_bool(value) diff --git a/patroni/dcs/zookeeper.py b/patroni/dcs/zookeeper.py index 3093ba06..3704b579 100644 --- a/patroni/dcs/zookeeper.py +++ b/patroni/dcs/zookeeper.py @@ -115,7 +115,8 @@ class ZooKeeper(AbstractDCS): self._client = PatroniKazooClient(hosts, handler=PatroniSequentialThreadingHandler(config['retry_timeout']), timeout=config['ttl'], connection_retry=KazooRetry(max_delay=1, max_tries=-1, sleep_func=time.sleep), command_retry=KazooRetry(max_delay=1, max_tries=-1, - deadline=config['retry_timeout'], sleep_func=time.sleep), **kwargs) + deadline=config['retry_timeout'], sleep_func=time.sleep), + auth_data=list(config.get('auth_data', {}).items()), **kwargs) self.__last_member_data: Optional[Dict[str, Any]] = None diff --git a/patroni/validator.py b/patroni/validator.py index 10bc7b2a..016f4b2e 100644 --- a/patroni/validator.py +++ b/patroni/validator.py @@ -1042,7 +1042,8 @@ schema = Schema({ Optional("key"): str, Optional("key_password"): str, Optional("verify"): bool, - Optional("set_acls"): dict + Optional("set_acls"): dict, + Optional("auth_data"): dict, }, "kubernetes": { "labels": {}, From d72f7cb259b54189bc3dec3ed26c7e2c9f6fff8a Mon Sep 17 00:00:00 2001 From: Israel Date: Wed, 1 Nov 2023 10:02:04 -0300 Subject: [PATCH 06/19] Add a FAQ page to the docs (#2933) This commit introduces a FAQ page to the docs. The idea is to get most frequently asked questions answered before-hand, so the user is able to get them answered quickly without going into detail in the docs or having to go to Slack/GitHub to clarify questions. --------- Signed-off-by: Israel Barth Rubio --- docs/contributing_guidelines.rst | 4 + docs/faq.rst | 329 +++++++++++++++++++++++++++++++ docs/index.rst | 1 + docs/patroni_configuration.rst | 4 + docs/rest_api.rst | 3 + 5 files changed, 341 insertions(+) create mode 100644 docs/faq.rst diff --git a/docs/contributing_guidelines.rst b/docs/contributing_guidelines.rst index 805f2fa8..48d2b77a 100644 --- a/docs/contributing_guidelines.rst +++ b/docs/contributing_guidelines.rst @@ -3,11 +3,15 @@ Contributing guidelines ======================= +.. _chatting: + Chatting -------- If you have a question, looking for an interactive troubleshooting help or want to chat with other Patroni users, join us on channel `#patroni `__ in the `PostgreSQL Slack `__. +.. _reporting_bugs: + Reporting bugs -------------- diff --git a/docs/faq.rst b/docs/faq.rst new file mode 100644 index 00000000..c56bf0ce --- /dev/null +++ b/docs/faq.rst @@ -0,0 +1,329 @@ +.. _faq: + +FAQ +=== + +In this section you will find answers for the most frequently asked questions about Patroni. +Each sub-section attempts to focus on different kinds of questions. + +We hope that this helps you to clarify most of your questions. +If you still have further concerns or find yourself facing an unexpected issue, please refer to :ref:`chatting` and :ref:`reporting_bugs` for instructions on how to get help or report issues. + +Comparison with other HA solutions +---------------------------------- + +Why does Patroni require a separate cluster of DCS nodes while other solutions like ``repmgr`` do not? + There are different ways of implementing HA solutions, each of them with their pros and cons. + + Software like ``repmgr`` performs communication among the nodes to decide when actions should be taken. + + Patroni on the other hand relies on the state stored in the DCS. The DCS acts as a source of truth for Patroni to decide what it should do. + + While having a separate DCS cluster can make you bloat your architecture, this approach also makes it less likely for split-brain scenarios to happen in your Postgres cluster. + +What is the difference between Patroni and other HA solutions in regards to Postgres management? + Patroni does not just manage the high availability of the Postgres cluster but also manages Postgres itself. + + If Postgres nodes do not exist yet, it takes care of bootstrapping the primary and the standby nodes, and also manages Postgres configuration of the nodes. If the Postgres nodes already exist, Patroni will take over management of the cluster. + + Besides the above, Patroni also has self-healing capabilities. In other words, if a primary node fails, Patroni will not only fail over to a replica, but also attempt to rejoin the former primary as a replica of the new primary. Similarly, if a replica fails, Patroni will attempt to rejoin that replica. + + That is way we call Patroni as a "template for HA solutions". It goes further than just managing physical replication: it manages Postgres as a whole. + +DCS +--- + +Can I use the same ``etcd`` cluster to store data from two or more Patroni clusters? + Yes, you can! + + Information about a Patroni cluster is stored in the DCS under a path prefixed with the ``namespace`` and ``scope`` Patroni settings. + + As long as you do not have conflicting namespace and scope across different Patroni clusters, you should be able to use the same DCS cluster to store information from multiple Patroni clusters. + +What occurs if I attempt to use the same combination of ``namespace`` and ``scope`` for different Patroni clusters that point to the same DCS cluster? + The second Patroni cluster that attempts to use the same ``namespace`` and ``scope`` will not be able to manage Postgres because it will find information related with that same combination in the DCS, but with an incompatible Postgres system identifier. + The mismatch on the system identifier causes Patroni to abort the management of the second cluster, as it assumes that refers to a different cluster and that the user has misconfigured Patroni. + + Make sure to use different ``namespace`` / ``scope`` when dealing with different Patroni clusters that share the same DCS cluster. + +What occurs if I lose my DCS cluster? + The DCS is used to store basically status and the dynamic configuration of the Patroni cluster. + + They very first consequence is that all the Patroni clusters that rely on that DCS will go to read-only mode -- unless :ref:`dcs_failsafe_mode` is enabled. + +What should I do if I lose my DCS cluster? + There are three possible outcomes upon losing your DCS cluster: + + 1. The DCS cluster is fully recovered: this requires no action from the Patroni side. Once the DCS cluster is recovered, Patroni should be able to recover too; + 2. The DCS cluster is re-created in place, and the endpoints remain the same. No changes are required on the Patroni side; + 3. A new DCS cluster is created with different endpoints. You will need to update the DCS endpoints in the Patroni configuration of each Patroni node. + + If you face scenario ``2.`` or ``3.`` Patroni will take care of creating the status information again based on the current status of the cluster, and recreate the dynamic configuration on the DCS based on a backup file named ``patroni.dynamic.json`` which is stored inside the Postgres data directory of each member of the Patroni cluster. + +What occurs if I lose majority in my DCS cluster? + The DCS will become unresponsive, which will cause Patroni to demote the current read/write Postgres node. + + Remember: Patroni relies on the state of the DCS to take actions on the cluster. + + You can use the :ref:`dcs_failsafe_mode` to alleviate that situation. + +patronictl +---------- + +Do I need to run :ref:`patronictl` in the Patroni host? + No, you do not need to do that. + + Running :ref:`patronictl` in the Patroni host is handy if you have access to the Patroni host because you can use the very same configuration file from the ``patroni`` agent for the :ref:`patronictl` application. + + However, :ref:`patronictl` is basically a client and it can be executed from remote machines. You just need to provide it with enough configuration so it can reach the DCS and the REST API of the Patroni member(s). + +Why did the information from one of my Patroni members disappear from the output of :ref:`patronictl_list` command? + Information shown by :ref:`patronictl_list` is based on the contents of the DCS. + + If information about a member disappeared from the DCS it is very likely that the Patroni agent on that node is not running anymore, or it is not able to communicate with the DCS. + + As the member is not able to update the information, the information eventually expires from the DCS, and consequently the member is not shown anymore in the output of :ref:`patronictl_list`. + +Why is the information about one of my Patroni members not up-to-date in the output of :ref:`patronictl_list` command? + Information shown by :ref:`patronictl_list` is based on the contents of the DCS. + + By default, that information is updated by Patroni roughly every ``loop_wait`` seconds. + In other words, even if everything is normally functional you may still see a "delay" of up to ``loop_wait`` seconds in the information stored in the DCS. + + Be aware that that is not a rule, though. Some operations performed by Patroni cause it to immediately update the DCS information. + +Configuration +------------- + +What is the difference between dynamic configuration and local configuration? + Dynamic configuration (or global configuration) is the configuration stored in the DCS, and which is applied to all members of the Patroni cluster. + This is primarily where you should store your configuration. + + Settings that are specific to a node, or settings that you would like to overwrite the global configuration with, you should set only on the desired Patroni member as a local configuration. + That local configuration can be specified either through the configuration file or through environment variables. + + See more in :ref:`patroni_configuration`. + +What are the types of configuration in Patroni, and what is the precedence? + The types are: + + * Dynamic configuration: applied to all members; + * Local configuration: applied to the local member, overrides dynamic configuration; + * Environment configuration: applied to the local member, overrides both dynamic and local configuration. + + **Note:** some Postgres GUCs can only be set globally, i.e., through dynamic configuration. Besides that, there are GUCs which Patroni enforces a hard-coded value. + + See more in :ref:`patroni_configuration`. + +Is there any facility to help me create my Patroni configuration file? + Yes, there is. + + You can use ``patroni --generate-sample-config`` or ``patroni --generate-config`` commands to generate a sample Patroni configuration or a Patroni configuration based on an existing Postgres instance, respectively. + + Please refer to :ref:`generate_sample_config` and :ref:`generate_config` for more details. + +I changed my parameters under ``bootstrap.dcs`` configuration but Patroni is not applying the changes to the cluster members. What is wrong? + The values configured under ``bootstrap.dcs`` are only used when bootstrapping a fresh cluster. Those values will be written to the DCS during the bootstrap. + + After the bootstrap phase finishes, you will only be able to change the dynamic configuration through the DCS. + + Refer to the next question for more details. + +How can I change my dynamic configuration? + You need to change the configuration in the DCS. That is accomplished either through: + + * :ref:`patronictl_edit_config`; or + * A ``PATCH`` request to :ref:`config_endpoint`. + +How can I change my local configuration? + You need to change the configuration file of the corresponding Patroni member and signal the Patroni agent with ``SIHGUP``. You can do that using either of these approaches: + + * Send a ``POST`` request to the REST API :ref:`reload_endpoint`; or + * Run :ref:`patronictl_reload`; or + * Locally signal the Patroni process with ``SIGHUP``: + + * If you started Patroni through systemd, you can use the command ``systemctl reload PATRONI_UNIT.service``, ``PATRONI_UNIT`` being the name of the Patroni service; or + * If you started Patroni through other means, you will need to identify the ``patroni`` process and run ``kill -s HUP PID``, ``PID`` being the process ID of the ``patroni`` process. + + **Note:** there are cases where a reload through the :ref:`patronictl_reload` may not work: + + * Expired REST API certificates: you can mitigate that by using the ``-k`` option of the :ref:`patronictl`; + * Wrong credentials: for example when changing ``restapi`` or ``ctl`` credentials in the configuration file, and using that same configuration file for Patroni and :ref:`patronictl`. + +How can I change my environment configuration? + The environment configuration is only read by Patroni during startup. + + With that in mind, if you change the environment configuration you will need to restart the corresponding Patroni agent. + + Take care to not cause a failover in the cluster! You might be interested in checking :ref:`patronictl_pause`. + +What occurs if I change a Postgres GUC that requires a reload? + When you change the dynamic or the local configuration as explained in the previous questions, Patroni will take care of reloading the Postgres configuration for you. + +What occurs if I change a Postgres GUC that requires a restart? + Patroni will mark the affected members with a flag of ``pending restart``. + + It is up to you to determine when and how to restart the members. That can be accomplished either through: + + * :ref:`patronictl_restart`; or + * A ``POST`` request to :ref:`restart_endpoint`. + + **Note:** some Postgres GUCs require a special management in terms of the order for restarting the Postgres nodes. Refer to :ref:`shared_memory_gucs` for more details. + +What is the difference between ``etcd`` and ``etcd3`` in Patroni configuration? + ``etcd`` uses the API version 2 of ``etcd``, while ``etcd3`` uses the API version 3 of ``etcd``. + + Be aware that information stored by the API version 2 is not manageable by API version 3 and vice-versa. + + We recommend that you configure ``etcd3`` instead of ``etcd`` because: + + * API version 2 is disabled by default from Etcd v3.4 onward; + * API version 2 will be completely removed on Etcd v3.6. + +I have ``use_slots`` enabled in my Patroni configuration, but when a cluster member goes offline for some time, the replication slot used by that member is dropped on the upstream node. What can I do to avoid that issue? + You can configure a permanent physical replication slot for the members. + + Since Patroni ``3.2.0`` it is now possible to have member slots as permanent slots managed by Patroni. + + Patroni will create the permanent physical slots on all nodes, and make sure to not remove the slots, as well as to advance the slots' LSN on all nodes according to the LSN that has been consumed by the member. + + Later, if you decide to remove the corresponding member, it's **your responsability** to adjust the permanent slots configuration, otherwise Patroni will keep the slots around forever. + + **Note:** on Patroni older than ``3.2.0`` you could still have member slots configured as permanent physical slots, however they would be managed only on the current leader. That is, in case of failover/switchover these slots would be created on the new leader, but that wouldn't guarantee that it had all WAL segments for the absent node. + + **Note:** even with Patroni ``3.2.0`` there might be a small race condition. In the very beginning, when the slot is created on the replica it could be ahead of the same slot on the leader and in case if nobody is consuming the slot there is still a chance that some files could be missing after failover. With that in mind, it is recommended that you configure continuous archiving, which makes it possible to restore required WALs or perform PITR. + +What is the difference between ``loop_wait``, ``retry_timeout`` and ``ttl``? + Patroni performs what we call a HA cycle from time to time. On each HA cycle it takes care of performing a series of checks on the cluster to determine its healthiness, and depending on the status it may take actions, like failing over to a standby. + + ``loop_wait`` determines for how long, in seconds, Patroni should sleep before performing a new cycle of HA checks. + + ``retry_timeout`` sets the timeout for retry operations on the DCS and on Postgres. For example: if the DCS is unresponsive for more than ``retry_timeout`` seconds, Patroni might demote the primary node as a security action. + + ``ttl`` sets the lease time on the ``leader`` lock in the DCS. If the current leader of the cluster is not able to renew the lease during its HA cycles for longer than ``ttl``, then the lease will expire and that will trigger a ``leader race`` in the cluster. + + **Note:** when modifying these settings, please keep in mind that Patroni enforces the rule and minimal values described in :ref:`dynamic_configuration` section of the docs. + +Postgres management +------------------- + +Can I change Postgres GUCs directly in Postgres configuration? + You can, but you should avoid that. + + Postgres configuration is managed by Patroni, and attempts to edit the configuration files may end up being frustrated by Patroni as it may eventually overwrite them. + + There are a few options available to overcome the management performed by Patroni: + + * Change Postgres GUCs through ``$PGDATA/postgresql.base.conf``; or + * Define a ``postgresql.custom_conf`` which will be used instead of ``postgresql.base.conf`` so you can manage that externally; or + * Change GUCs using ``ALTER SYSTEM`` / ``ALTER DATABASE`` / ``ALTER USER``. + + You can find more information about that in the section :ref:`important_configuration_rules`. + + In any case we recommend that you manage all the Postgres configuration through Patroni. That will centralize the management and make it easier to debug Patroni when needed. + +Can I restart Postgres nodes directly? + No, you should **not** attempt to manage Postgres directly! + + Any attempt of bouncing the Postgres server without Patroni can lead your cluster to face failovers. + + If you need to manage the Postgres server, do that through the ways exposed by Patroni. + +Is Patroni able to take over management of an already existing Postgres cluster? + Yes, it can! + + Please refer to :ref:`existing_data` for detailed instructions. + +How does Patroni manage Postgres? + Patroni takes care of bringing Postgres up and down by running the Postgres binaries, like ``pg_ctl`` and ``postgres``. + + With that in mind you **MUST** disable any other sources that could manage the Postgres clusters, like the systemd units, e.g. ``postgresql.service``. Only Patroni should be able to start, stop and promote Postgres instances in the cluster. Not doing so may result in split-brain scenarios. For example: if the node running as a primary failed and the unit ``postgresql.service`` is enabled, it may bring Postgres back up and cause a split-brain. + +Concepts and requirements +------------------------- + +Which are the applications that make part of Patroni? + Patroni basically ships a couple applications: + + * ``patroni``: This is the Patroni agent, which takes care of managing a Postgres node; + * ``patronictl``: This is a command-line utility used to interact with a Patroni cluster (perform switchovers, restarts, changes in the configuration, etc.). Please find more information in :ref:`patronictl`. + +What is a ``standby cluster`` in Patroni? + It is a cluster that does not have any primary Postgres node running, i.e., there is no read/write member in the cluster. + + These kinds of clusters exist to replicate data from another cluster and are usually useful when you want to replicate data across data centers. + + There will be a leader in the cluster which will be a standby in charge of replicating changes from a remote Postgres node. + Then, there will be a set of standbys configured with cascading replication from such leader member. + + **Note:** the standby cluster doesn't know anything about the source cluster which it is replicating from -- it can even use ``restore_command`` instead of WAL streaming, and may use an absolutely independent DCS cluster. + + Refer to :ref:`standby_cluster` for more details. + +What is a ``leader`` in Patroni? + A ``leader`` in Patroni is like a coordinator of the cluster. + + In a regular Patroni cluster, the ``leader`` will be the read/write node. + + In a standby Patroni cluster, the ``leader`` (AKA ``standby leader``) will be in charge of replicating from a remote Postgres node, and cascading those changes to the other members of the standby cluster. + +Does Patroni require a minimum number of Postgres nodes in the cluster? + No, you can run Patroni with any number of Postgres nodes. + + Remember: Patroni is decoupled from the DCS. + +What does ``pause`` mean in Patroni? + Pause is an operation exposed by Patroni so the user can ask Patroni to step back in regards to Postgres management. + + That is mainly useful when you want to perform maintenance on the cluster, and would like to avoid that Patroni takes decisions related with HA, like failing over to a standby when you stop the primary. + + You can find more information about that in :ref:`pause`. + +Automatic failover +------------------ + +How does the automatic failover mechanism of Patroni work? + Patroni automatic failover is based on what we call ``leader race``. + + Patroni stores the cluster's status in the DCS, among them a ``leader`` lock which holds the name of the Patroni member which is the current ``leader`` of the cluster. + + That ``leader`` lock has a time-to-live associated with it. If the leader node fails to update the lease of the ``leader`` lock in time, the key will eventually expire from the DCS. + + When the ``leader`` lock expires, it triggers what Patroni calls a ``leader race``: all nodes start performing checks to determine if they are the best candidates for taking over the ``leader`` role. + Some of these checks include calls to the REST API of all other Patroni members. + + All Patroni members that find themselves as the best candidate for taking over the ``leader`` lock will attempt to do so. + The first Patroni member that is able to take the ``leader`` lock will promote itself to a read/write node (or ``standby leader``), and the others will be configured to follow it. + +Can I temporarily disable automatic failover in the Patroni cluster? + Yes, you can! + + You can achieve that by temporarily pausing the cluster. + This is typically useful for performing maintenance. + + When you want to resume the automatic failover of the cluster, you just need to unpause it. + + You can find more information about that in :ref:`pause`. + +Bootstrapping and standbys creation +----------------------------------- + +How does Patroni create a primary Postgres node? What about a standby Postgres node? + By default Patroni will use ``initdb`` to bootstrap a fresh cluster, and ``pg_basebackup`` to create standby nodes from a copy of the ``leader`` member. + + You can customize that behavior by writing your custom bootstrap methods, and your custom replica creation methods. + + Custom methods are usually useful when you want to restore backups created by backup tools like pgBackRest or Barman, for example. + + For detailed information please refer to :ref:`custom_bootstrap` and :ref:`custom_replica_creation`. + +Monitoring +---------- + +How can I monitor my Patroni cluster? + Patroni exposes a couple handy endpoints in its :ref:`rest_api`: + + * ``/metrics``: exposes monitoring metrics in a format that can be consumed by Prometheus; + * ``/patroni``: exposes the status of the cluster in a JSON format. The information shown here is very similar to what is shown by the ``/metrics`` endpoint. + + You can use those endpoints to implement monitoring checks. diff --git a/docs/index.rst b/docs/index.rst index f5a4f2d4..84d02db1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -36,6 +36,7 @@ Currently supported PostgreSQL versions: 9.3 to 16. existing_data security ha_multi_dc + faq releases CONTRIBUTING diff --git a/docs/patroni_configuration.rst b/docs/patroni_configuration.rst index 1d6c0777..06d71426 100644 --- a/docs/patroni_configuration.rst +++ b/docs/patroni_configuration.rst @@ -30,6 +30,7 @@ There are 3 types of Patroni configuration: It is possible to set/override some of the "Local" configuration parameters with environment variables. Environment configuration is very useful when you are running in a dynamic environment and you don't know some of the parameters in advance (for example it's not possible to know your external IP address when you are running inside ``docker``). +.. _important_configuration_rules: Important rules --------------- @@ -90,6 +91,7 @@ The parameters would be applied in the following order (run-time are given the h This allows configuration for all the nodes (2), configuration for a specific node using ``ALTER SYSTEM`` (3) and ensures that parameters essential to the running of Patroni are enforced (4), as well as leaves room for configuration tools that manage `postgresql.conf` directly without involving Patroni (1). +.. _shared_memory_gucs: PostgreSQL parameters that touch shared memory ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -156,6 +158,7 @@ Patroni provides command-line interfaces for a Patroni :ref:`local configuration - Create a Patroni configuration file for the locally running PostgreSQL instance (e.g. as a preparation step for the :ref:`Patroni integration `); - Validate a given Patroni configuration file. +.. _generate_sample_config: Sample Patroni configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -183,6 +186,7 @@ Parameters ``configfile`` - full path to the configuration file used to store the result. If not provided, the result is sent to ``stdout``. +.. _generate_config: Patroni configuration for a running instance ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/rest_api.rst b/docs/rest_api.rst index 00272100..a03c05d2 100644 --- a/docs/rest_api.rst +++ b/docs/rest_api.rst @@ -426,6 +426,7 @@ Cluster status endpoints ] ] +.. _config_endpoint: Config endpoint --------------- @@ -666,6 +667,7 @@ There are a couple of checks that a member of a cluster should pass to be able t - its lag exceeds the maximum replication lag allowed; - it has the timeline number smaller than the last known cluster timeline. +.. _restart_endpoint: Restart endpoint ---------------- @@ -682,6 +684,7 @@ Restart endpoint ``POST /restart`` and ``DELETE /restart`` endpoints are used by :ref:`patronictl_restart` and :ref:`patronictl flush cluster-name restart ` respectively. +.. _reload_endpoint: Reload endpoint --------------- From 8adddb3467f3c43ddf4ff723a2381e0cf6e2a31b Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 3 Nov 2023 13:02:39 +0100 Subject: [PATCH 07/19] Limit accepted values for --format argument (#2938) It used to accept any arbitrary string Close https://github.com/zalando/patroni/issues/2936 --- patroni/ctl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/patroni/ctl.py b/patroni/ctl.py index 83810dd3..b4c57873 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -255,7 +255,8 @@ def load_config(path: str, dcs_url: Optional[str]) -> Dict[str, Any]: return config -option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, tsv, json, yaml)', default='pretty') +option_format = click.option('--format', '-f', 'fmt', help='Output format', default='pretty', + type=click.Choice(['pretty', 'tsv', 'json', 'yaml', 'yml'])) option_watchrefresh = click.option('-w', '--watch', type=float, help='Auto update the screen every X seconds') option_watch = click.option('-W', is_flag=True, help='Auto update the screen every 2 seconds') option_force = click.option('--force', is_flag=True, help='Do not ask for confirmation at any point') From 269b04be5d0ede9332cc9f79199b15f287ac37b0 Mon Sep 17 00:00:00 2001 From: Israel Date: Mon, 6 Nov 2023 12:25:27 -0300 Subject: [PATCH 08/19] Add a contrib script for remote Barman recovery (#2931) A contrib script, which can be used as a custom bootstrap method, or as a custom create replica method. The script communicates with the pg-backup-api on the Barman node so Patroni is able to restore a Barman backup remotely. The `--help` option of the script, along with the script docstring, should provide some context on how to use fill its parameters. Patroni docs were updated accordingly to share examples about how to configure the script as a custom bootstrap method, or as a custom create replica method. References: PAT-216. --- docs/replica_bootstrap.rst | 35 +++ patroni/scripts/barman_recover.py | 468 ++++++++++++++++++++++++++++++ setup.py | 3 +- tests/test_barman_recover.py | 364 +++++++++++++++++++++++ 4 files changed, 869 insertions(+), 1 deletion(-) create mode 100644 patroni/scripts/barman_recover.py create mode 100644 tests/test_barman_recover.py diff --git a/docs/replica_bootstrap.rst b/docs/replica_bootstrap.rst index 861a0e86..a6e3db8e 100644 --- a/docs/replica_bootstrap.rst +++ b/docs/replica_bootstrap.rst @@ -71,6 +71,22 @@ Makes the configured ``command`` to be called additionally with ``--arg1=value1 .. note:: Bootstrap methods are neither chained, nor fallen-back to the default one in case the primary one fails +As an example, you are able to bootstrap a fresh Patroni cluster from a Barman backup with a configuration like this: + +.. code:: YAML + + bootstrap: + method: barman + barman: + keep_existing_recovery_conf: true + command: patroni_barman_recover + api-url: https://barman-host:7480 + barman-server: my_server + ssh-command: ssh postgres@patroni-host + +.. note:: + ``patroni_barman_recover`` requires that you have both Barman and ``pg-backup-api`` configured in the Barman host, so it can execute a remote ``barman recover`` through the backup API. + The above example uses a subset of the available parameters. You can get more information running ``patroni_barman_recover --help``. .. _custom_replica_creation: @@ -125,6 +141,25 @@ example: pgbackrest basebackup: max-rate: '100M' +example: Barman + +.. code:: YAML + + postgresql: + create_replica_methods: + - barman + - basebackup + barman: + command: patroni_barman_recover + api-url: https://barman-host:7480 + barman-server: my_server + ssh-command: ssh postgres@patroni-host + basebackup: + max-rate: '100M' + +.. note:: + ``patroni_barman_recover`` requires that you have both Barman and ``pg-backup-api`` configured in the Barman host, so it can execute a remote ``barman recover`` through the backup API. + The above example uses a subset of the available parameters. You can get more information running ``patroni_barman_recover --help``. The ``create_replica_methods`` defines available replica creation methods and the order of executing them. Patroni will stop on the first one that returns 0. Each method should define a separate section in the configuration file, listing the command diff --git a/patroni/scripts/barman_recover.py b/patroni/scripts/barman_recover.py new file mode 100644 index 00000000..1cffe34d --- /dev/null +++ b/patroni/scripts/barman_recover.py @@ -0,0 +1,468 @@ +#!/usr/bin/env python + +"""Restore a Barman backup to the local node through ``pg-backup-api``. + +This script can be used both as a custom bootstrap method, and as a custom +create replica method. Check the output of ``--help`` to understand the +parameters supported by the script. ``--datadir`` is a special parameter and it +is automatically filled by Patroni in both cases. + +It requires that you have previously configured a Barman server, and that you +have ``pg-backup-api`` configured and running in the same host as Barman. + +Refer to :class:`ExitCode` for possible exit codes of this script. +""" +from argparse import ArgumentParser +from enum import IntEnum +import json +import logging +import sys +import time +from typing import Any, Callable, Optional, Tuple, Type, Union +from urllib.parse import urljoin +from urllib3 import PoolManager +from urllib3.exceptions import MaxRetryError +from urllib3.response import HTTPResponse + + +class ExitCode(IntEnum): + """Possible exit codes of this script. + + :cvar RECOVERY_DONE: backup was successfully restored. + :cvar RECOVERY_FAILED: recovery of the backup faced an issue. + :cvar API_NOT_OK: ``pg-backup-api`` status is not ``OK``. + :cvar HTTP_REQUEST_ERROR: an error has occurred during a request to the + ``pg-backup-api``. + :cvar HTTP_RESPONSE_MALFORMED: ``pg-backup-api`` returned a bogus response. + """ + + RECOVERY_DONE = 0 + RECOVERY_FAILED = 1 + API_NOT_OK = 2 + HTTP_REQUEST_ERROR = 3 + HTTP_RESPONSE_MALFORMED = 4 + + +class RetriesExceeded(Exception): + """Maximum number of retries exceeded.""" + + +def retry(exceptions: Union[Type[Exception], Tuple[Type[Exception], ...]]) \ + -> Any: + """Retry an operation n times if expected *exceptions* are faced. + + .. note:: + Should be used as a decorator of a class' method as it expects the + first argument to be a class instance. + + The class which method is going to be decorated should contain a couple + attributes: + + * ``max_retries``: maximum retry attempts before failing; + * ``retry_wait``: how long to wait before retrying. + + :param exceptions: exceptions that could trigger a retry attempt. + + :raises: + :exc:`RetriesExceeded`: if the maximum number of attempts has been + exhausted. + """ + def decorator(func: Callable[..., Any]) -> Any: + def inner_func(instance: object, *args: Any, **kwargs: Any) -> Any: + times: int = getattr(instance, "max_retries") + retry_wait: int = getattr(instance, "retry_wait") + method_name = f"{instance.__class__.__name__}.{func.__name__}" + + attempt = 1 + + while attempt <= times: + try: + return func(instance, *args, **kwargs) + except exceptions as exc: + logging.warning("Attempt %d of %d on method %s failed " + "with %r.", + attempt, times, method_name, exc) + attempt += 1 + + time.sleep(retry_wait) + + raise RetriesExceeded("Maximum number of retries exceeded for " + f"method {method_name}.") + return inner_func + return decorator + + +class BarmanRecover: + """Facilities for performing a remote ``barman recover`` operation. + + You should instantiate this class, which will take care of configuring the + operation accordingly. When you want to start the operation, you should + call :meth:`restore_backup`. At any point of interaction with this class, + you may face a :func:`sys.exit` call. Refer to :class:`ExitCode` for a view + on the possible exit codes. + + :ivar api_url: base URL to reach the ``pg-backup-api``. + :ivar cert_file: certificate to authenticate against the + ``pg-backup-api``, if required. + :ivar key_file: certificate key to authenticate against the + ``pg-backup-api``, if required. + :ivar barman_server: name of the Barman server which backup is to be + restored. + :ivar backup_id: ID of the backup from the Barman server. + :ivar ssh_command: SSH command to connect from the Barman host to the + local host. + :ivar data_directory: path to the Postgres data directory where to + restore the backup at. + :ivar loop_wait: how long to wait before checking again the status of the + recovery process. Higher values are useful for backups that are + expected to take long to restore. + :ivar retry_wait: how long to wait before retrying a failed request to the + ``pg-backup-api``. + :ivar max_retries: maximum number of retries when ``pg-backup-api`` returns + malformed responses. + :ivar http: a HTTP pool manager for performing web requests. + """ + + def __init__(self, api_url: str, barman_server: str, backup_id: str, + ssh_command: str, data_directory: str, loop_wait: int, + retry_wait: int, max_retries: int, + cert_file: Optional[str] = None, + key_file: Optional[str] = None) -> None: + """Create a new instance of :class:`BarmanRecover`. + + Make sure the ``pg-backup-api`` is reachable and running fine. + + :param api_url: base URL to reach the ``pg-backup-api``. + :param barman_server: name of the Barman server which backup is to be + restored. + :param backup_id: ID of the backup from the Barman server. + :param ssh_command: SSH command to connect from the Barman host to the + local host. + :param data_directory: path to the Postgres data directory where to + restore the backup at. + :param loop_wait: how long to wait before checking again the status of + the recovery process. Higher values are useful for backups that are + expected to take long to restore. + :param retry_wait: how long to wait before retrying a failed request to + the ``pg-backup-api``. + :param max_retries: maximum number of retries when ``pg-backup-api`` + returns malformed responses. + :param cert_file: certificate to authenticate against the + ``pg-backup-api``, if required. + :param key_file: certificate key to authenticate against the + ``pg-backup-api``, if required. + """ + self.api_url = api_url + self.cert_file = cert_file + self.key_file = key_file + self.barman_server = barman_server + self.backup_id = backup_id + self.ssh_command = ssh_command + self.data_directory = data_directory + self.loop_wait = loop_wait + self.retry_wait = retry_wait + self.max_retries = max_retries + self.http = PoolManager(cert_file=cert_file, key_file=key_file) + self._ensure_api_ok() + + def _build_full_url(self, url_path: str) -> str: + """Build the full URL by concatenating *url_path* with the base URL. + + :param url_path: path to be accessed in the ``pg-backup-api``. + + :returns: the full URL after concatenating. + """ + return urljoin(self.api_url, url_path) + + @staticmethod + def _deserialize_response(response: HTTPResponse) -> Any: + """Retrieve body from *response* as a deserialized JSON object. + + :param response: response from which JSON body will be deserialized. + + :returns: the deserialized JSON body. + """ + return json.loads(response.data.decode("utf-8")) + + @staticmethod + def _serialize_request(body: Any) -> Any: + """Serialize a request body. + + :param body: content of the request body to be serialized. + + :returns: the serialized request body. + """ + return json.dumps(body).encode("utf-8") + + def _get_request(self, url_path: str) -> Any: + """Perform a ``GET`` request to *url_path*. + + .. note:: + If a :exc:`MaxRetryError` is faced while performing the request, + then exit with :attr:`ExitCode.HTTP_REQUEST_ERROR` + + :param url_path: URL to perform the ``GET`` request against. + + :returns: the deserialized response body. + """ + response = None + + try: + response = self.http.request("GET", self._build_full_url(url_path)) + except MaxRetryError as exc: + logging.critical("An error occurred while performing an HTTP GET " + "request: %r", exc) + sys.exit(ExitCode.HTTP_REQUEST_ERROR) + + return self._deserialize_response(response) + + def _post_request(self, url_path: str, body: Any) -> Any: + """Perform a ``POST`` request to *url_path* serializing *body* as JSON. + + .. note:: + If a :exc:`MaxRetryError` is faced while performing the request, + then exit with :attr:`ExitCode.HTTP_REQUEST_ERROR` + + :param url_path: URL to perform the ``POST`` request against. + :param body: the body to be serialized as JSON and sent in the request. + + :returns: the deserialized response body. + """ + body = self._serialize_request(body) + + response = None + + try: + response = self.http.request("POST", + self._build_full_url(url_path), + body=body, + headers={ + "Content-Type": "application/json" + }) + except MaxRetryError as exc: + logging.critical("An error occurred while performing an HTTP POST " + "request: %r", exc) + sys.exit(ExitCode.HTTP_REQUEST_ERROR) + + return self._deserialize_response(response) + + def _ensure_api_ok(self) -> None: + """Ensure ``pg-backup-api`` is reachable and ``OK``. + + .. note:: + If ``pg-backup-api`` status is not ``OK``, then exit with + :attr:`ExitCode.API_NOT_OK`. + """ + response = self._get_request("status") + + if response != "OK": + logging.critical("pg-backup-api is not working: %s", response) + sys.exit(ExitCode.API_NOT_OK) + + @retry(KeyError) + def _create_recovery_operation(self) -> str: + """Create a recovery operation on the ``pg-backup-api``. + + :returns: the ID of the recovery operation that has been created. + """ + response = self._post_request( + f"servers/{self.barman_server}/operations", + { + "type": "recovery", + "backup_id": self.backup_id, + "remote_ssh_command": self.ssh_command, + "destination_directory": self.data_directory, + }, + ) + + return response["operation_id"] + + @retry(KeyError) + def _get_recovery_operation_status(self, operation_id: str) -> str: + """Get status of the recovery operation *operation_id*. + + :param operation_id: ID of the recovery operation to be checked. + + :returns: the status of the recovery operation. + """ + response = self._get_request( + f"servers/{self.barman_server}/operations/{operation_id}", + ) + + return response["status"] + + def restore_backup(self) -> bool: + """Restore the configured Barman backup through ``pg-backup-api``. + + .. note:: + If recovery API request returns a malformed response, then exit with + :attr:`ExitCode.HTTP_RESPONSE_MALFORMED`. + + :returns: ``True`` if it was successfully recovered, ``False`` + otherwise. + """ + operation_id = None + + try: + operation_id = self._create_recovery_operation() + except RetriesExceeded: + logging.critical("Maximum number of retries exceeded, exiting.") + sys.exit(ExitCode.HTTP_RESPONSE_MALFORMED) + + logging.info("Created the recovery operation with ID %s", operation_id) + + status = None + + while True: + try: + status = self._get_recovery_operation_status(operation_id) + except RetriesExceeded: + logging.critical("Maximum number of retries exceeded, " + "exiting.") + sys.exit(ExitCode.HTTP_RESPONSE_MALFORMED) + + if status != "IN_PROGRESS": + break + + logging.info("Recovery operation %s is still in progress", + operation_id) + time.sleep(self.loop_wait) + + return status == "DONE" + + +def set_up_logging(log_file: Optional[str] = None) -> None: + """Set up logging to file, if *log_file* is given, otherwise to console. + + :param log_file: file where to log messages, if any. + """ + logging.basicConfig(filename=log_file, level=logging.INFO, + format="%(asctime)s %(levelname)s: %(message)s") + + +def main() -> None: + """Entry point of this script. + + Parse the command-line arguments and recover a Barman backup through + ``pg-backup-api`` to the local host. + """ + parser = ArgumentParser( + epilog=( + "Wrapper script for ``pg-backup-api``. Communicate with the API " + "running at ``--api-url`` to restore a ``--backup-id`` Barman " + "backup of the server ``--barman-server``." + ), + ) + parser.add_argument( + "--api-url", + type=str, + required=True, + help="URL to reach the ``pg-backup-api``, e.g. " + "``http://localhost:7480``", + dest="api_url", + ) + parser.add_argument( + "--cert-file", + type=str, + required=False, + help="Certificate to authenticate against the API, if required.", + dest="cert_file", + ) + parser.add_argument( + "--key-file", + type=str, + required=False, + help="Certificate key to authenticate against the API, if required.", + dest="key_file", + ) + parser.add_argument( + "--barman-server", + type=str, + required=True, + help="Name of the Barman server from which to restore the backup.", + dest="barman_server", + ) + parser.add_argument( + "--backup-id", + type=str, + required=False, + default="latest", + help="ID of the Barman backup to be restored. You can use any value " + "supported by ``barman recover`` command " + "(default: ``%(default)s``)", + dest="backup_id", + ) + parser.add_argument( + "--ssh-command", + type=str, + required=True, + help="Value to be passed as ``--remote-ssh-command`` to " + "``barman recover``.", + dest="ssh_command", + ) + parser.add_argument( + "--data-directory", + "--datadir", + type=str, + required=True, + help="Destination path where to restore the barman backup in the " + "local host.", + dest="data_directory", + ) + parser.add_argument( + "--log-file", + type=str, + required=False, + help="File where to log messages produced by this script, if any.", + dest="log_file", + ) + parser.add_argument( + "--loop-wait", + type=int, + required=False, + default=10, + help="How long to wait before checking again the status of the " + "recovery process, in seconds. Use higher values if your " + "recovery is expected to take long (default: ``%(default)s``)", + dest="loop_wait", + ) + parser.add_argument( + "--retry-wait", + type=int, + required=False, + default=2, + help="How long to wait before retrying a failed ``pg-backup-api`` " + "request (default: ``%(default)s``)", + dest="retry_wait", + ) + parser.add_argument( + "--max-retries", + type=int, + required=False, + default=5, + help="Maximum number of retries when receiving malformed responses " + "from the ``pg-backup-api`` (default: ``%(default)s``)", + dest="max_retries", + ) + args, _ = parser.parse_known_args() + + set_up_logging(args.log_file) + + barman_recover = BarmanRecover(args.api_url, args.barman_server, + args.backup_id, args.ssh_command, + args.data_directory, args.loop_wait, + args.retry_wait, args.max_retries, + args.cert_file, args.key_file) + + successful = barman_recover.restore_backup() + + if successful: + logging.info("Recovery operation finished successfully.") + sys.exit(ExitCode.RECOVERY_DONE) + else: + logging.critical("Recovery operation failed.") + sys.exit(ExitCode.RECOVERY_FAILED) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 4c1c25b3..5bd9c71f 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,8 @@ CONSOLE_SCRIPTS = ['patroni = patroni.__main__:main', 'patronictl = patroni.ctl:ctl', 'patroni_raft_controller = patroni.raft_controller:main', "patroni_wale_restore = patroni.scripts.wale_restore:main", - "patroni_aws = patroni.scripts.aws:main"] + "patroni_aws = patroni.scripts.aws:main", + "patroni_barman_recover = patroni.scripts.barman_recover:main"] class _Command(Command): diff --git a/tests/test_barman_recover.py b/tests/test_barman_recover.py new file mode 100644 index 00000000..4334d6ae --- /dev/null +++ b/tests/test_barman_recover.py @@ -0,0 +1,364 @@ +import logging +from mock import MagicMock, Mock, call, patch +import unittest +from urllib3.exceptions import MaxRetryError + +from patroni.scripts.barman_recover import BarmanRecover, ExitCode, RetriesExceeded, main, set_up_logging + + +API_URL = "http://localhost:7480" +BARMAN_SERVER = "my_server" +BACKUP_ID = "backup_id" +SSH_COMMAND = "ssh postgres@localhost" +DATA_DIRECTORY = "/path/to/pgdata" +LOOP_WAIT = 10 +RETRY_WAIT = 2 +MAX_RETRIES = 5 + + +class TestBarmanRecover(unittest.TestCase): + + @patch.object(BarmanRecover, "_ensure_api_ok", Mock()) + @patch("patroni.scripts.barman_recover.PoolManager", MagicMock()) + def setUp(self): + self.br = BarmanRecover(API_URL, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT, RETRY_WAIT, + MAX_RETRIES) + # Reset the mock as the same instance is used across tests + self.br.http.request.reset_mock() + self.br.http.request.side_effect = None + + def test__build_full_url(self): + self.assertEqual(self.br._build_full_url("/some/path"), f"{API_URL}/some/path") + + @patch("json.loads") + def test__deserialize_response(self, mock_json_loads): + mock_response = MagicMock() + self.assertIsNotNone(self.br._deserialize_response(mock_response)) + mock_json_loads.assert_called_once_with(mock_response.data.decode("utf-8")) + + @patch("json.dumps") + def test__serialize_request(self, mock_json_dumps): + body = "some_body" + ret = self.br._serialize_request(body) + self.assertIsNotNone(ret) + mock_json_dumps.assert_called_once_with(body) + mock_json_dumps.return_value.encode.assert_called_once_with("utf-8") + + @patch.object(BarmanRecover, "_deserialize_response", Mock(return_value="test")) + @patch("logging.critical") + def test__get_request(self, mock_logging): + mock_request = self.br.http.request + + # with no error + self.assertEqual(self.br._get_request("/some/path"), "test") + mock_request.assert_called_once_with("GET", f"{API_URL}/some/path") + + # with MaxRetryError + http_error = MaxRetryError(self.br.http, f"{API_URL}/some/path") + mock_request.side_effect = http_error + + with self.assertRaises(SystemExit) as exc: + self.assertIsNone(self.br._get_request("/some/path")) + + mock_logging.assert_called_once_with("An error occurred while performing an HTTP GET request: %r", http_error) + self.assertEqual(exc.exception.code, ExitCode.HTTP_REQUEST_ERROR) + + # with Exception + mock_logging.reset_mock() + mock_request.side_effect = Exception("Some error.") + + with patch("sys.exit") as mock_sys: + with self.assertRaises(Exception): + self.assertIsNone(self.br._get_request("/some/path")) + + mock_logging.assert_not_called() + mock_sys.assert_not_called() + + @patch.object(BarmanRecover, "_deserialize_response", Mock(return_value="test")) + @patch("logging.critical") + @patch.object(BarmanRecover, "_serialize_request") + def test__post_request(self, mock_serialize, mock_logging): + mock_request = self.br.http.request + + # with no error + self.assertEqual(self.br._post_request("/some/path", "some body"), "test") + mock_serialize.assert_called_once_with("some body") + mock_request.assert_called_once_with("POST", f"{API_URL}/some/path", body=mock_serialize.return_value, + headers={"Content-Type": "application/json"}) + + # with HTTPError + http_error = MaxRetryError(self.br.http, f"{API_URL}/some/path") + mock_request.side_effect = http_error + + with self.assertRaises(SystemExit) as exc: + self.assertIsNone(self.br._post_request("/some/path", "some body")) + + mock_logging.assert_called_once_with("An error occurred while performing an HTTP POST request: %r", http_error) + self.assertEqual(exc.exception.code, ExitCode.HTTP_REQUEST_ERROR) + + # with Exception + mock_logging.reset_mock() + mock_request.side_effect = Exception("Some error.") + + with patch("sys.exit") as mock_sys: + with self.assertRaises(Exception): + self.br._post_request("/some/path", "some body") + + mock_logging.assert_not_called() + mock_sys.assert_not_called() + + @patch("logging.critical") + @patch.object(BarmanRecover, "_get_request") + def test__ensure_api_ok(self, mock_get_request, mock_logging): + # API ok + mock_get_request.return_value = "OK" + + with patch("sys.exit") as mock_sys: + self.assertIsNone(self.br._ensure_api_ok()) + mock_logging.assert_not_called() + mock_sys.assert_not_called() + + # API not ok + mock_get_request.return_value = "random" + + with self.assertRaises(SystemExit) as exc: + self.assertIsNone(self.br._ensure_api_ok()) + + mock_logging.assert_called_once_with("pg-backup-api is not working: %s", "random") + self.assertEqual(exc.exception.code, ExitCode.API_NOT_OK) + + @patch("logging.warning") + @patch("time.sleep") + @patch.object(BarmanRecover, "_post_request") + def test__create_recovery_operation(self, mock_post_request, mock_sleep, mock_logging): + # well formed response + mock_post_request.return_value = {"operation_id": "some_id"} + self.assertEqual(self.br._create_recovery_operation(), "some_id") + mock_sleep.assert_not_called() + mock_logging.assert_not_called() + mock_post_request.assert_called_once_with( + f"servers/{BARMAN_SERVER}/operations", + { + "type": "recovery", + "backup_id": BACKUP_ID, + "remote_ssh_command": SSH_COMMAND, + "destination_directory": DATA_DIRECTORY, + } + ) + + # malformed response + mock_post_request.return_value = {"operation_idd": "some_id"} + + with self.assertRaises(RetriesExceeded) as exc: + self.br._create_recovery_operation() + + self.assertEqual(str(exc.exception), + "Maximum number of retries exceeded for method BarmanRecover._create_recovery_operation.") + + self.assertEqual(mock_sleep.call_count, self.br.max_retries) + mock_sleep.assert_has_calls([call(self.br.retry_wait)] * self.br.max_retries) + + self.assertEqual(mock_logging.call_count, self.br.max_retries) + for i in range(mock_logging.call_count): + call_args = mock_logging.mock_calls[i].args + self.assertEqual(len(call_args), 5) + self.assertEqual(call_args[0], "Attempt %d of %d on method %s failed with %r.") + self.assertEqual(call_args[1], i + 1) + self.assertEqual(call_args[2], self.br.max_retries) + self.assertEqual(call_args[3], "BarmanRecover._create_recovery_operation") + self.assertIsInstance(call_args[4], KeyError) + self.assertEqual(repr(call_args[4]), "KeyError('operation_id')") + + @patch("logging.warning") + @patch("time.sleep") + @patch.object(BarmanRecover, "_get_request") + def test__get_recovery_operation_status(self, mock_get_request, mock_sleep, mock_logging): + # well formed response + mock_get_request.return_value = {"status": "some status"} + self.assertEqual(self.br._get_recovery_operation_status("some_id"), "some status") + mock_get_request.assert_called_once_with(f"servers/{BARMAN_SERVER}/operations/some_id") + mock_sleep.assert_not_called() + mock_logging.assert_not_called() + + # malformed response + mock_get_request.return_value = {"statuss": "some status"} + + with self.assertRaises(RetriesExceeded) as exc: + self.br._get_recovery_operation_status("some_id") + + self.assertEqual(str(exc.exception), + "Maximum number of retries exceeded for method BarmanRecover._get_recovery_operation_status.") + + self.assertEqual(mock_sleep.call_count, self.br.max_retries) + mock_sleep.assert_has_calls([call(self.br.retry_wait)] * self.br.max_retries) + + self.assertEqual(mock_logging.call_count, self.br.max_retries) + for i in range(mock_logging.call_count): + call_args = mock_logging.mock_calls[i].args + self.assertEqual(len(call_args), 5) + self.assertEqual(call_args[0], "Attempt %d of %d on method %s failed with %r.") + self.assertEqual(call_args[1], i + 1) + self.assertEqual(call_args[2], self.br.max_retries) + self.assertEqual(call_args[3], "BarmanRecover._get_recovery_operation_status") + self.assertIsInstance(call_args[4], KeyError) + self.assertEqual(repr(call_args[4]), "KeyError('status')") + + @patch.object(BarmanRecover, "_get_recovery_operation_status") + @patch("time.sleep") + @patch("logging.info") + @patch("logging.critical") + @patch.object(BarmanRecover, "_create_recovery_operation") + def test_restore_backup(self, mock_create_op, mock_log_critical, mock_log_info, mock_sleep, mock_get_status): + # successful fast restore + mock_create_op.return_value = "some_id" + mock_get_status.return_value = "DONE" + + self.assertTrue(self.br.restore_backup()) + + mock_create_op.assert_called_once() + mock_get_status.assert_called_once_with("some_id") + mock_log_info.assert_called_once_with("Created the recovery operation with ID %s", "some_id") + mock_log_critical.assert_not_called() + mock_sleep.assert_not_called() + + # successful slow restore + mock_create_op.reset_mock() + mock_get_status.reset_mock() + mock_log_info.reset_mock() + mock_get_status.side_effect = ["IN_PROGRESS"] * 20 + ["DONE"] + + self.assertTrue(self.br.restore_backup()) + + mock_create_op.assert_called_once() + + self.assertEqual(mock_get_status.call_count, 21) + mock_get_status.assert_has_calls([call("some_id")] * 21) + + self.assertEqual(mock_log_info.call_count, 21) + mock_log_info.assert_has_calls([call("Created the recovery operation with ID %s", "some_id")] + + [call("Recovery operation %s is still in progress", "some_id")] * 20) + + mock_log_critical.assert_not_called() + + self.assertEqual(mock_sleep.call_count, 20) + mock_sleep.assert_has_calls([call(LOOP_WAIT)] * 20) + + # failed fast restore + mock_create_op.reset_mock() + mock_get_status.reset_mock() + mock_log_info.reset_mock() + mock_sleep.reset_mock() + mock_get_status.side_effect = None + mock_get_status.return_value = "FAILED" + + self.assertFalse(self.br.restore_backup()) + + mock_create_op.assert_called_once() + mock_get_status.assert_called_once_with("some_id") + mock_log_info.assert_called_once_with("Created the recovery operation with ID %s", "some_id") + mock_log_critical.assert_not_called() + mock_sleep.assert_not_called() + + # failed slow restore + mock_create_op.reset_mock() + mock_get_status.reset_mock() + mock_log_info.reset_mock() + mock_sleep.reset_mock() + mock_get_status.side_effect = ["IN_PROGRESS"] * 20 + ["FAILED"] + + self.assertFalse(self.br.restore_backup()) + + mock_create_op.assert_called_once() + + self.assertEqual(mock_get_status.call_count, 21) + mock_get_status.assert_has_calls([call("some_id")] * 21) + + self.assertEqual(mock_log_info.call_count, 21) + mock_log_info.assert_has_calls([call("Created the recovery operation with ID %s", "some_id")] + + [call("Recovery operation %s is still in progress", "some_id")] * 20) + + mock_log_critical.assert_not_called() + + self.assertEqual(mock_sleep.call_count, 20) + mock_sleep.assert_has_calls([call(LOOP_WAIT)] * 20) + + # create retries exceeded + mock_log_info.reset_mock() + mock_sleep.reset_mock() + mock_create_op.side_effect = RetriesExceeded + mock_get_status.side_effect = None + + with self.assertRaises(SystemExit) as exc: + self.assertIsNone(self.br.restore_backup()) + + self.assertEqual(exc.exception.code, ExitCode.HTTP_RESPONSE_MALFORMED) + mock_log_info.assert_not_called() + mock_log_critical.assert_called_once_with("Maximum number of retries exceeded, exiting.") + mock_sleep.assert_not_called() + + # get status retries exceeded + mock_create_op.reset_mock() + mock_create_op.side_effect = None + mock_log_critical.reset_mock() + mock_log_info.reset_mock() + mock_get_status.side_effect = RetriesExceeded + + with self.assertRaises(SystemExit) as exc: + self.assertIsNone(self.br.restore_backup()) + + self.assertEqual(exc.exception.code, ExitCode.HTTP_RESPONSE_MALFORMED) + mock_log_info.assert_called_once_with("Created the recovery operation with ID %s", "some_id") + mock_log_critical.assert_called_once_with("Maximum number of retries exceeded, exiting.") + mock_sleep.assert_not_called() + + +class TestMain(unittest.TestCase): + + @patch("logging.basicConfig") + def test_set_up_logging(self, mock_log_config): + log_file = "/path/to/some/file.log" + set_up_logging(log_file) + mock_log_config.assert_called_once_with(filename=log_file, level=logging.INFO, + format="%(asctime)s %(levelname)s: %(message)s") + + @patch("logging.critical") + @patch("logging.info") + @patch("patroni.scripts.barman_recover.set_up_logging") + @patch("patroni.scripts.barman_recover.BarmanRecover") + @patch("patroni.scripts.barman_recover.ArgumentParser") + def test_main(self, mock_arg_parse, mock_br, mock_set_up_log, mock_log_info, mock_log_critical): + # successful restore + args = MagicMock() + mock_arg_parse.return_value.parse_known_args.return_value = (args, None) + mock_br.return_value.restore_backup.return_value = True + + with self.assertRaises(SystemExit) as exc: + main() + + mock_arg_parse.assert_called_once() + mock_set_up_log.assert_called_once_with(args.log_file) + mock_br.assert_called_once_with(args.api_url, args.barman_server, args.backup_id, args.ssh_command, + args.data_directory, args.loop_wait, args.retry_wait, args.max_retries, + args.cert_file, args.key_file) + mock_log_info.assert_called_once_with("Recovery operation finished successfully.") + mock_log_critical.assert_not_called() + self.assertEqual(exc.exception.code, ExitCode.RECOVERY_DONE) + + # failed restore + mock_arg_parse.reset_mock() + mock_set_up_log.reset_mock() + mock_br.reset_mock() + mock_log_info.reset_mock() + mock_br.return_value.restore_backup.return_value = False + + with self.assertRaises(SystemExit) as exc: + main() + + mock_arg_parse.assert_called_once() + mock_set_up_log.assert_called_once_with(args.log_file) + mock_br.assert_called_once_with(args.api_url, args.barman_server, args.backup_id, args.ssh_command, + args.data_directory, args.loop_wait, args.retry_wait, args.max_retries, + args.cert_file, args.key_file) + mock_log_info.assert_not_called() + mock_log_critical.assert_called_once_with("Recovery operation failed.") + self.assertEqual(exc.exception.code, ExitCode.RECOVERY_FAILED) From 552e8643d9d5555ecaf7a882638bcc8302dc6933 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 7 Nov 2023 11:05:54 +0100 Subject: [PATCH 09/19] Verify that replica nodes received checkpoint LSN on shutdown (#2939) In case if archiving is enabled the `Postgresql.latest_checkpoint_location()` method returns LSN of the prev (SWITCH) record, which points to the beginning of the WAL file. It is done in order to make it possible to safely promote replica which recovers WAL files from the archive and wasn't streaming when the primary was stopped (primary doesn't archive this WAL file). But, in certain cases using the LSN pointing to SWITCH record was causing unnecessary pg_rewind, if replica didn't managed to replay shutdown checkpoint record before it was promoted. In order to mitigate the problem we need to check that replica received/replayed exactly the shutdown checkpoint LSN. But, at the same time we will still write LSN of the SWITCH record to the `/status` key when releasing the leader lock. --- patroni/ha.py | 13 ++++++----- patroni/postgresql/__init__.py | 40 ++++++++++++++++++++++++---------- tests/test_ha.py | 2 +- tests/test_postgresql.py | 5 ++++- 4 files changed, 41 insertions(+), 19 deletions(-) diff --git a/patroni/ha.py b/patroni/ha.py index 233ed4b2..4a21972a 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -1227,15 +1227,16 @@ class Ha(object): status = {'released': False} - def on_shutdown(checkpoint_location: int) -> None: + def on_shutdown(checkpoint_location: int, prev_location: int) -> None: # Postmaster is still running, but pg_control already reports clean "shut down". # It could happen if Postgres is still archiving the backlog of WAL files. # If we know that there are replicas that received the shutdown checkpoint # location, we can remove the leader key and allow them to start leader race. + time.sleep(1) # give replicas some more time to catch up if self.is_failover_possible(cluster_lsn=checkpoint_location): self.state_handler.set_role('demoted') with self._async_executor: - self.release_leader_key_voluntarily(checkpoint_location) + self.release_leader_key_voluntarily(prev_location) status['released'] = True def before_shutdown() -> None: @@ -1990,18 +1991,18 @@ class Ha(object): status = {'deleted': False} - def _on_shutdown(checkpoint_location: int) -> None: + def _on_shutdown(checkpoint_location: int, prev_location: int) -> None: if self.is_leader(): # Postmaster is still running, but pg_control already reports clean "shut down". # It could happen if Postgres is still archiving the backlog of WAL files. # If we know that there are replicas that received the shutdown checkpoint # location, we can remove the leader key and allow them to start leader race. - + time.sleep(1) # give replicas some more time to catch up if self.is_failover_possible(cluster_lsn=checkpoint_location): - self.dcs.delete_leader(self.cluster.leader, checkpoint_location) + self.dcs.delete_leader(self.cluster.leader, prev_location) status['deleted'] = True else: - self.dcs.write_leader_optime(checkpoint_location) + self.dcs.write_leader_optime(prev_location) def _before_shutdown() -> None: self.notify_citus_coordinator('before_demote') diff --git a/patroni/postgresql/__init__.py b/patroni/postgresql/__init__.py index 7a3d4065..fdfc26c1 100644 --- a/patroni/postgresql/__init__.py +++ b/patroni/postgresql/__init__.py @@ -591,14 +591,17 @@ class Postgresql(object): return match.group(1), match.group(2), match.group(3), match.group(4) return None, None, None, None - def latest_checkpoint_location(self) -> Optional[int]: - """Returns checkpoint location for the cleanly shut down primary. - But, if we know that the checkpoint was written to the new WAL - due to the archive_mode=on, we will return the LSN of prev wal record (SWITCH).""" + def _checkpoint_locations_from_controldata(self, data: Dict[str, str]) -> Optional[Tuple[int, int]]: + """Get shutdown checkpoint location. - data = self.controldata() + :param data: :class:`dict` object with values returned by `pg_controldata` tool. + + :returns: a tuple of checkpoint LSN for the cleanly shut down primary, and LSN of prev wal record (SWITCH) + if we know that the checkpoint was written to the new WAL file due to the archive_mode=on. + """ timeline = data.get("Latest checkpoint's TimeLineID") lsn = checkpoint_lsn = data.get('Latest checkpoint location') + prev_lsn = None if data.get('Database cluster state') == 'shut down' and lsn and timeline and checkpoint_lsn: try: checkpoint_lsn = parse_lsn(checkpoint_lsn) @@ -609,13 +612,26 @@ class Postgresql(object): _, lsn, _, desc = self.parse_wal_record(timeline, prev) prev = parse_lsn(prev) # If the cluster is shutdown with archive_mode=on, WAL is switched before writing the checkpoint. - # In this case we want to take the LSN of previous record (switch) as the last known WAL location. + # In this case we want to take the LSN of previous record (SWITCH) as the last known WAL location. if lsn and parse_lsn(lsn) == prev and str(desc).strip() in ('xlog switch', 'SWITCH'): - return prev + prev_lsn = prev except Exception as e: logger.error('Exception when parsing WAL pg_%sdump output: %r', self.wal_name, e) if isinstance(checkpoint_lsn, int): - return checkpoint_lsn + return checkpoint_lsn, (prev_lsn or checkpoint_lsn) + + def latest_checkpoint_location(self) -> Optional[int]: + """Get shutdown checkpoint location. + + .. note:: + In case if checkpoint was written to the new WAL file due to the archive_mode=on + we return LSN of the previous wal record (SWITCH). + + :returns: checkpoint LSN for the cleanly shut down primary. + """ + checkpoint_locations = self._checkpoint_locations_from_controldata(self.controldata()) + if checkpoint_locations: + return checkpoint_locations[1] def is_running(self) -> Optional[PostmasterProcess]: """Returns PostmasterProcess if one is running on the data directory or None. If most recently seen process @@ -801,7 +817,7 @@ class Postgresql(object): return 'not accessible or not healty' def stop(self, mode: str = 'fast', block_callbacks: bool = False, checkpoint: Optional[bool] = None, - on_safepoint: Optional[Callable[..., Any]] = None, on_shutdown: Optional[Callable[[int], Any]] = None, + on_safepoint: Optional[Callable[..., Any]] = None, on_shutdown: Optional[Callable[[int, int], Any]] = None, before_shutdown: Optional[Callable[..., Any]] = None, stop_timeout: Optional[int] = None) -> bool: """Stop PostgreSQL @@ -831,7 +847,7 @@ class Postgresql(object): return success def _do_stop(self, mode: str, block_callbacks: bool, checkpoint: bool, - on_safepoint: Optional[Callable[..., Any]], on_shutdown: Optional[Callable[..., Any]], + on_safepoint: Optional[Callable[..., Any]], on_shutdown: Optional[Callable[[int, int], Any]], before_shutdown: Optional[Callable[..., Any]], stop_timeout: Optional[int]) -> Tuple[bool, bool]: postmaster = self.is_running() if not postmaster: @@ -871,7 +887,9 @@ class Postgresql(object): while postmaster.is_running(): data = self.controldata() if data.get('Database cluster state', '') == 'shut down': - on_shutdown(self.latest_checkpoint_location()) + checkpoint_locations = self._checkpoint_locations_from_controldata(data) + if checkpoint_locations: + on_shutdown(*checkpoint_locations) break elif data.get('Database cluster state', '').startswith('shut down'): # shut down in recovery break diff --git a/tests/test_ha.py b/tests/test_ha.py index 45675e72..e197d28b 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -1532,7 +1532,7 @@ class TestHa(PostgresInit): self.ha.is_leader = true def stop(*args, **kwargs): - kwargs['on_shutdown'](123) + kwargs['on_shutdown'](123, 120) self.p.stop = stop self.ha.shutdown() diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py index 6ae69e85..cdd0c160 100644 --- a/tests/test_postgresql.py +++ b/tests/test_postgresql.py @@ -237,7 +237,10 @@ class TestPostgresql(BaseTestPostgresql): @patch.object(Postgresql, 'latest_checkpoint_location', Mock(return_value='7')) def test__do_stop(self): mock_callback = Mock() - with patch.object(Postgresql, 'controldata', Mock(return_value={'Database cluster state': 'shut down'})): + with patch.object(Postgresql, 'controldata', + Mock(return_value={'Database cluster state': 'shut down', + "Latest checkpoint's TimeLineID": '1', + 'Latest checkpoint location': '1/1'})): self.assertTrue(self.p.stop(on_shutdown=mock_callback, stop_timeout=3)) mock_callback.assert_called() with patch.object(Postgresql, 'controldata', From 3ffd598a1c559f9d9bb3b7e43cd8a24454c3f3de Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 8 Nov 2023 14:08:02 +0100 Subject: [PATCH 10/19] Do a real http request when performing name uniqueness check (#2942) When running in containers it is possible that the traffic is routed using `docker-proxy`, which listens on the port and accepting incoming connections. This commit effectively sticks to the original solution from #2878 --- patroni/__main__.py | 16 +++++++--------- patroni/log.py | 29 +++++++++++++++++++++-------- tests/test_patroni.py | 8 ++++---- 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/patroni/__main__.py b/patroni/__main__.py index 02ba56da..229ccfb9 100644 --- a/patroni/__main__.py +++ b/patroni/__main__.py @@ -107,8 +107,6 @@ class Patroni(AbstractPatroniDaemon, Tags): def ensure_unique_name(self) -> None: """A helper method to prevent splitbrain from operator naming error.""" - from urllib.parse import urlparse - from urllib3.connection import HTTPConnection from patroni.dcs import Member cluster = self.dcs.get_cluster() @@ -118,14 +116,14 @@ class Patroni(AbstractPatroniDaemon, Tags): if not isinstance(member, Member): return try: - parts = urlparse(member.api_url) - if isinstance(parts.hostname, str): - connection = HTTPConnection(parts.hostname, port=parts.port or 80, timeout=3) - connection.connect() - logger.fatal("Can't start; there is already a node named '%s' running", self.config['name']) - sys.exit(1) + # Silence annoying WARNING: Retrying (...) messages when Patroni is quickly restarted. + # At this moment we don't have custom log levels configured and hence shouldn't lose anything useful. + self.logger.update_loggers({'urllib3.connectionpool': 'ERROR'}) + _ = self.request(member, endpoint="/liveness", timeout=3) + logger.fatal("Can't start; there is already a node named '%s' running", self.config['name']) + sys.exit(1) except Exception: - return + self.logger.update_loggers({}) def _get_tags(self) -> Dict[str, Any]: """Get tags configured for this node, if any. diff --git a/patroni/log.py b/patroni/log.py index 09d73883..6ac67a17 100644 --- a/patroni/log.py +++ b/patroni/log.py @@ -202,24 +202,37 @@ class PatroniLogger(Thread): self._proxy_handler = ProxyHandler(self) self._root_logger.addHandler(self._proxy_handler) - def update_loggers(self) -> None: - """Configure loggers' log level as defined in ``log.loggers`` section of Patroni configuration. + def update_loggers(self, config: Dict[str, Any]) -> None: + """Configure custom loggers' log levels. .. note:: It creates logger objects that are not defined yet in the log manager. + + :param config: :class:`dict` object with custom loggers configuration, is set either from: + + * ``log.loggers`` section of Patroni configuration; or + + * from the method that is trying to make sure that the node name + isn't duplicated (to silence annoying ``urllib3`` WARNING's). + + :Example: + + .. code-block:: python + + update_loggers({'urllib3.connectionpool': 'WARNING'}) """ - loggers = deepcopy((self._config or {}).get('loggers') or {}) + loggers = deepcopy(config) for name, logger in self._root_logger.manager.loggerDict.items(): # ``Placeholder`` is a node in the log manager for which no logger has been defined. We are interested only # in the ones that were defined if not isinstance(logger, logging.PlaceHolder): - # if this logger is present in ``log.loggers`` Patroni configuration, use the configured level, - # otherwise use ``logging.NOTSET``, which means it will inherit the level from any parent node up to - # the root for which log level is defined. + # if this logger is present in *config*, use the configured level, otherwise + # use ``logging.NOTSET``, which means it will inherit the level + # from any parent node up to the root for which log level is defined. level = loggers.pop(name, logging.NOTSET) logger.setLevel(level) - # define loggers that do not exist yet and set level as configured in ``log.loggers`` section of configuration. + # define loggers that do not exist yet and set level as configured in the *config* for name, level in loggers.items(): logger = self._root_logger.manager.getLogger(name) logger.setLevel(level) @@ -274,7 +287,7 @@ class PatroniLogger(Thread): self.log_handler = new_handler self._config = config.copy() - self.update_loggers() + self.update_loggers(config.get('loggers') or {}) def _close_old_handlers(self) -> None: """Close old log handlers. diff --git a/tests/test_patroni.py b/tests/test_patroni.py index 19497ab5..bf9e2871 100644 --- a/tests/test_patroni.py +++ b/tests/test_patroni.py @@ -45,7 +45,7 @@ class MockFrozenImporter(object): @patch('time.sleep', Mock()) @patch('subprocess.call', Mock(return_value=0)) @patch('patroni.psycopg.connect', psycopg_connect) -@patch('urllib3.connection.HTTPConnection.connect', Mock(side_effect=Exception)) +@patch('urllib3.PoolManager.request', Mock(side_effect=Exception)) @patch.object(ConfigHandler, 'append_pg_hba', Mock()) @patch.object(ConfigHandler, 'write_postgresql_conf', Mock()) @patch.object(ConfigHandler, 'write_recovery_conf', Mock()) @@ -69,7 +69,7 @@ class TestPatroni(unittest.TestCase): self.assertRaises(SystemExit, _main) @patch('pkgutil.iter_importers', Mock(return_value=[MockFrozenImporter()])) - @patch('urllib3.connection.HTTPConnection.connect', Mock(side_effect=Exception)) + @patch('urllib3.PoolManager.request', Mock(side_effect=Exception)) @patch('sys.frozen', Mock(return_value=True), create=True) @patch.object(HTTPServer, '__init__', Mock()) @patch.object(etcd.Client, 'read', etcd_read) @@ -273,8 +273,8 @@ class TestPatroni(unittest.TestCase): ) with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=bad_cluster)): # If the api of the running node cannot be reached, this implies unique name - with patch('urllib3.connection.HTTPConnection.connect', Mock(side_effect=ConnectionError)): + with patch('urllib3.PoolManager.request', Mock(side_effect=ConnectionError)): self.assertIsNone(self.p.ensure_unique_name()) # Only if the api of the running node is reachable do we throw an error - with patch('urllib3.connection.HTTPConnection.connect', Mock()): + with patch('urllib3.PoolManager.request', Mock()): self.assertRaises(SystemExit, self.p.ensure_unique_name) From 1b96ae9c0ad47fc1c6ee64c8c184b53061bb50ad Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 9 Nov 2023 11:09:38 +0100 Subject: [PATCH 11/19] Fix Etcd v2 with Citus (#2943) When deploying a new Citus cluster with Etcd v2 Patroni was failing to start with the following exception: ```python 2023-11-09 10:51:41,246 INFO: Selected new etcd server http://localhost:2379 Traceback (most recent call last): File "/home/akukushkin/git/patroni/./patroni.py", line 6, in main() File "/home/akukushkin/git/patroni/patroni/__main__.py", line 343, in main return patroni_main(args.configfile) File "/home/akukushkin/git/patroni/patroni/__main__.py", line 237, in patroni_main abstract_main(Patroni, configfile) File "/home/akukushkin/git/patroni/patroni/daemon.py", line 172, in abstract_main controller = cls(config) File "/home/akukushkin/git/patroni/patroni/__main__.py", line 66, in __init__ self.ensure_unique_name() File "/home/akukushkin/git/patroni/patroni/__main__.py", line 112, in ensure_unique_name cluster = self.dcs.get_cluster() File "/home/akukushkin/git/patroni/patroni/dcs/__init__.py", line 1654, in get_cluster cluster = self._get_citus_cluster() if self.is_citus_coordinator() else self.__get_patroni_cluster() File "/home/akukushkin/git/patroni/patroni/dcs/__init__.py", line 1638, in _get_citus_cluster cluster = groups.pop(CITUS_COORDINATOR_GROUP_ID, Cluster.empty()) AttributeError: 'Cluster' object has no attribute 'pop' ``` It is broken since #2909. In addition to that fix `_citus_cluster_loader()` interface by allowing it to return only dict obj. --- patroni/dcs/__init__.py | 2 +- patroni/dcs/consul.py | 4 +--- patroni/dcs/etcd.py | 13 +++++++++---- tests/test_etcd.py | 2 ++ 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/patroni/dcs/__init__.py b/patroni/dcs/__init__.py index 389cae86..9c2f7335 100644 --- a/patroni/dcs/__init__.py +++ b/patroni/dcs/__init__.py @@ -1564,7 +1564,7 @@ class AbstractDCS(abc.ABC): """ @abc.abstractmethod - def _citus_cluster_loader(self, path: Any) -> Union[Cluster, Dict[int, Cluster]]: + def _citus_cluster_loader(self, path: Any) -> Dict[int, Cluster]: """Load and build all Patroni clusters from a single Citus cluster. :param path: the path in DCS where to load Cluster(s) from. diff --git a/patroni/dcs/consul.py b/patroni/dcs/consul.py index 58e20030..1324e606 100644 --- a/patroni/dcs/consul.py +++ b/patroni/dcs/consul.py @@ -422,7 +422,7 @@ class Consul(AbstractDCS): def _cluster_loader(self, path: str) -> Cluster: _, results = self.retry(self._client.kv.get, path, recurse=True, consistency=self._consistency) if results is None: - raise NotFound + return Cluster.empty() nodes = {} for node in results: node['Value'] = (node['Value'] or b'').decode('utf-8') @@ -445,8 +445,6 @@ class Consul(AbstractDCS): ) -> Union[Cluster, Dict[int, Cluster]]: try: return loader(path) - except NotFound: - return Cluster.empty() except Exception: logger.exception('get_cluster') raise ConsulError('Consul is not responding properly') diff --git a/patroni/dcs/etcd.py b/patroni/dcs/etcd.py index f242a6b2..3be699a6 100644 --- a/patroni/dcs/etcd.py +++ b/patroni/dcs/etcd.py @@ -710,13 +710,20 @@ class Etcd(AbstractEtcd): return Cluster(initialize, config, leader, status, members, failover, sync, history, failsafe) def _cluster_loader(self, path: str) -> Cluster: - result = self.retry(self._client.read, path, recursive=True, quorum=self._ctl) + try: + result = self.retry(self._client.read, path, recursive=True, quorum=self._ctl) + except etcd.EtcdKeyNotFound: + return Cluster.empty() nodes = {node.key[len(result.key):].lstrip('/'): node for node in result.leaves} return self._cluster_from_nodes(result.etcd_index, nodes) def _citus_cluster_loader(self, path: str) -> Dict[int, Cluster]: + try: + result = self.retry(self._client.read, path, recursive=True, quorum=self._ctl) + except etcd.EtcdKeyNotFound: + return {} + clusters: Dict[int, Dict[str, etcd.EtcdResult]] = defaultdict(dict) - result = self.retry(self._client.read, path, recursive=True, quorum=self._ctl) for node in result.leaves: key = node.key[len(result.key):].lstrip('/').split('/', 1) if len(key) == 2 and citus_group_re.match(key[0]): @@ -729,8 +736,6 @@ class Etcd(AbstractEtcd): cluster = None try: cluster = loader(path) - except etcd.EtcdKeyNotFound: - cluster = Cluster.empty() except Exception as e: self._handle_exception(e, 'get_cluster', raise_ex=EtcdError('Etcd is not responding properly')) self._has_failed = False diff --git a/tests/test_etcd.py b/tests/test_etcd.py index 90402b5f..874aac5c 100644 --- a/tests/test_etcd.py +++ b/tests/test_etcd.py @@ -274,6 +274,8 @@ class TestEtcd(unittest.TestCase): cluster = self.etcd.get_cluster() self.assertIsInstance(cluster, Cluster) self.assertIsInstance(cluster.workers[1], Cluster) + self.etcd._base_path = '/service/nocluster' + self.assertTrue(self.etcd.get_cluster().is_empty()) def test_touch_member(self): self.assertFalse(self.etcd.touch_member('')) From 7370f70f13f2db968ed860ea9e8137d275dee860 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 10 Nov 2023 09:23:45 +0100 Subject: [PATCH 12/19] Fix pg_rewind behavior with Postgres v16+ (#2944) The error message format was changed in https://github.com/postgres/postgres/commit/4ac30ba4f29d4b586b131404b0d514f16501272a, what caused `pg_rewind` being called by Patroni even when it was not necessary. --- patroni/postgresql/rewind.py | 41 ++++++++++++++++++++++++++---------- tests/test_rewind.py | 10 +++++++++ 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/patroni/postgresql/rewind.py b/patroni/postgresql/rewind.py index 6e1aab88..4a5283f7 100644 --- a/patroni/postgresql/rewind.py +++ b/patroni/postgresql/rewind.py @@ -101,12 +101,26 @@ class Rewind(object): return 'not accessible or not healty' def _get_checkpoint_end(self, timeline: int, lsn: int) -> int: - """The checkpoint record size in WAL depends on postgres major version and platform (memory alignment). - Hence, the only reliable way to figure out where it ends, read the record from file with the help of pg_waldump - and parse the output. We are trying to read two records, and expect that it will fail to read the second one: - `pg_waldump: fatal: error in WAL record at 0/182E220: invalid record length at 0/182E298: wanted 24, got 0` - The error message contains information about LSN of the next record, which is exactly where checkpoint ends.""" + """Get the end of checkpoint record from WAL. + .. note:: + The checkpoint record size in WAL depends on postgres major version and platform (memory alignment). + Hence, the only reliable way to figure out where it ends, is to read the record from file with the + help of ``pg_waldump`` and parse the output. + + We are trying to read two records, and expect that it will fail to read the second record with message: + + fatal: error in WAL record at 0/182E220: invalid record length at 0/182E298: wanted 24, got 0; or + + fatal: error in WAL record at 0/182E220: invalid record length at 0/182E298: expected at least 24, got 0 + + The error message contains information about LSN of the next record, which is exactly where checkpoint ends. + + :param timeline: the checkpoint *timeline* from ``pg_controldata``. + :param lsn: the checkpoint *location* as :class:`int` from ``pg_controldata``. + + :returns: the end of checkpoint record as :class:`int` or ``0`` if failed to parse ``pg_waldump`` output. + """ lsn8 = format_lsn(lsn, True) lsn_str = format_lsn(lsn) out, err = self._postgresql.waldump(timeline, lsn_str, 2) @@ -117,12 +131,17 @@ class Rewind(object): if len(out) == 1 and len(err) == 1 and ', lsn: {0}, prev '.format(lsn8) in out[0] and pattern in err[0]: i = err[0].find(pattern) + len(pattern) - j = err[0].find(": wanted ", i) - if j > -1: - try: - return parse_lsn(err[0][i:j]) - except Exception as e: - logger.error('Failed to parse lsn %s: %r', err[0][i:j], e) + # Message format depends on the major version: + # * expected at least -- starting from v16 + # * wanted -- before v16 + # We will simply check all possible combinations. + for pattern in (': expected at least ', ': wanted '): + j = err[0].find(pattern, i) + if j > -1: + try: + return parse_lsn(err[0][i:j]) + except Exception as e: + logger.error('Failed to parse lsn %s: %r', err[0][i:j], e) logger.error('Failed to parse pg_%sdump output', self._postgresql.wal_name) logger.error(' stdout=%s', '\n'.join(out)) logger.error(' stderr=%s', '\n'.join(err)) diff --git a/tests/test_rewind.py b/tests/test_rewind.py index a54c27e9..af8e7d97 100644 --- a/tests/test_rewind.py +++ b/tests/test_rewind.py @@ -180,6 +180,16 @@ class TestRewind(BaseTestPostgresql): self.r.trigger_check_diverged_lsn() mock_get_local_timeline_lsn.return_value = (False, 2, 67197377) self.assertTrue(self.r.rewind_or_reinitialize_needed_and_possible(self.leader)) + + mock_popen.return_value.communicate.return_value = ( + b'0, lsn: 0/040159C1, prev 0/\n', + b'pg_waldump: fatal: error in WAL record at 0/40159C1: invalid record ' + b'length at 0/402DD98: expected at least 24, got 0\n' + ) + self.r.reset_state() + self.r.trigger_check_diverged_lsn() + self.assertFalse(self.r.rewind_or_reinitialize_needed_and_possible(self.leader)) + self.r.reset_state() self.r.trigger_check_diverged_lsn() mock_popen.side_effect = Exception From 1870dcd8f954822201f6de46402dd2671163b1f9 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 13 Nov 2023 15:01:57 +0100 Subject: [PATCH 13/19] Fix bug with custom bootstrap (#2948) Patroni was falsely applying `--command` argument. Close https://github.com/zalando/patroni/issues/2947 --- patroni/postgresql/bootstrap.py | 7 +++---- tests/test_bootstrap.py | 9 ++++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/patroni/postgresql/bootstrap.py b/patroni/postgresql/bootstrap.py index 751c0797..a544bd73 100644 --- a/patroni/postgresql/bootstrap.py +++ b/patroni/postgresql/bootstrap.py @@ -188,10 +188,9 @@ class Bootstrap(object): params = [] if config.get('no_params') else ['--scope=' + self._postgresql.scope, '--datadir=' + self._postgresql.data_dir] # Add custom parameters specified by the user - reserved_args = {'no_params', 'keep_existing_recovery_conf', 'recovery_conf', 'scope', 'datadir'} - for arg, val in config.items(): - if arg not in reserved_args: - params.append(f"--{arg}={val}") + reserved_args = {'command', 'no_params', 'keep_existing_recovery_conf', 'recovery_conf', 'scope', 'datadir'} + params += [f"--{arg}={val}" for arg, val in config.items() if arg not in reserved_args] + try: logger.info('Running custom bootstrap script: %s', config['command']) if self._postgresql.cancellable.call(shlex.split(config['command']) + params) != 0: diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 4c2d1c98..8724b03c 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -179,10 +179,17 @@ class TestBootstrap(BaseTestPostgresql): @patch.object(Postgresql, 'controldata', Mock(return_value={'Database cluster state': 'in production'})) def test_custom_bootstrap(self, mock_cancellable_subprocess_call): self.p.config._config.pop('pg_hba') - config = {'method': 'foo', 'foo': {'command': 'bar'}} + config = {'method': 'foo', 'foo': {'command': 'bar --arg1=val1'}} mock_cancellable_subprocess_call.return_value = 1 self.assertFalse(self.b.bootstrap(config)) + self.assertEqual(mock_cancellable_subprocess_call.call_args_list[0][0][0], + ['bar', '--arg1=val1', '--scope=batman', '--datadir=' + os.path.join('data', 'test0')]) + + mock_cancellable_subprocess_call.reset_mock() + config['foo']['no_params'] = 1 + self.assertFalse(self.b.bootstrap(config)) + self.assertEqual(mock_cancellable_subprocess_call.call_args_list[0][0][0], ['bar', '--arg1=val1']) mock_cancellable_subprocess_call.return_value = 0 with patch('multiprocessing.Process', Mock(side_effect=Exception("42"))), \ From ecf158bce38095fb711e9c598c6090d3428a02b9 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 14 Nov 2023 13:44:54 +0100 Subject: [PATCH 14/19] Get rid of pass_obj() in most of patronictl commands (#2945) The `obj` could be easily obtained with the help of `click.get_current_context().obj`. Introduced function `is_citus_cluster()` will simplify future refactoring to add support of other MPP databases. In addition to that refactor ctl.py unit tests by moving most of mocks to the global scope., --- patroni/ctl.py | 236 ++++++++++++-------------- tests/test_ctl.py | 421 ++++++++++++++++++++-------------------------- 2 files changed, 286 insertions(+), 371 deletions(-) diff --git a/patroni/ctl.py b/patroni/ctl.py index b4c57873..2d92457a 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -255,15 +255,23 @@ def load_config(path: str, dcs_url: Optional[str]) -> Dict[str, Any]: return config +def _get_configuration() -> Dict[str, Any]: + """Get configuration object. + + :returns: configuration object from the current context. + """ + return click.get_current_context().obj['__config'] + + option_format = click.option('--format', '-f', 'fmt', help='Output format', default='pretty', type=click.Choice(['pretty', 'tsv', 'json', 'yaml', 'yml'])) option_watchrefresh = click.option('-w', '--watch', type=float, help='Auto update the screen every X seconds') option_watch = click.option('-W', is_flag=True, help='Auto update the screen every 2 seconds') option_force = click.option('--force', is_flag=True, help='Do not ask for confirmation at any point') arg_cluster_name = click.argument('cluster_name', required=False, - default=lambda: click.get_current_context().obj.get('scope')) + default=lambda: _get_configuration().get('scope')) option_default_citus_group = click.option('--group', required=False, type=int, help='Citus group', - default=lambda: click.get_current_context().obj.get('citus', {}).get('group')) + default=lambda: _get_configuration().get('citus', {}).get('group')) option_citus_group = click.option('--group', required=False, type=int, help='Citus group') role_choice = click.Choice(['leader', 'primary', 'standby-leader', 'replica', 'standby', 'any', 'master']) @@ -301,15 +309,23 @@ def ctl(ctx: click.Context, config_file: str, dcs_url: Optional[str], insecure: level = os.environ.get(name, level) logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=level) logging.captureWarnings(True) # Capture eventual SSL warning - ctx.obj = load_config(config_file, dcs_url) + config = load_config(config_file, dcs_url) # backward compatibility for configuration file where ctl section is not defined - ctx.obj.setdefault('ctl', {})['insecure'] = ctx.obj.get('ctl', {}).get('insecure') or insecure + config.setdefault('ctl', {})['insecure'] = config.get('ctl', {}).get('insecure') or insecure + ctx.obj = {'__config': config} -def get_dcs(config: Dict[str, Any], scope: str, group: Optional[int]) -> AbstractDCS: +def is_citus_cluster() -> bool: + """Check if we are working with Citus cluster. + + :returns: ``True`` if configuration has ``citus`` section, otherwise ``False``. + """ + return bool(_get_configuration().get('citus')) + + +def get_dcs(scope: str, group: Optional[int]) -> AbstractDCS: """Get the DCS object. - :param config: Patroni configuration. :param scope: cluster name. :param group: if *group* is defined, use it to select which alternative Citus group this DCS refers to. If *group* is ``None`` and a Citus configuration exists, assume this is the coordinator. Coordinator has the group ``0``. @@ -320,13 +336,14 @@ def get_dcs(config: Dict[str, Any], scope: str, group: Optional[int]) -> Abstrac :raises: :class:`PatroniCtlException`: if not suitable DCS configuration could be found. """ + config = _get_configuration() config.update({'scope': scope, 'patronictl': True}) if group is not None: config['citus'] = {'group': group} config.setdefault('name', scope) try: dcs = _get_dcs(config) - if config.get('citus') and group is None: + if is_citus_cluster() and group is None: dcs.is_citus_coordinator = lambda: True return dcs except PatroniException as e: @@ -347,7 +364,7 @@ def request_patroni(member: Member, method: str = 'GET', ctx = click.get_current_context() # the current click context request_executor = ctx.obj.get('__request_patroni') if not request_executor: - request_executor = ctx.obj['__request_patroni'] = PatroniRequest(ctx.obj) + request_executor = ctx.obj['__request_patroni'] = PatroniRequest(_get_configuration()) return request_executor(member, method, endpoint, data) @@ -452,11 +469,9 @@ def watching(w: bool, watch: Optional[int], max_count: Optional[int] = None, cle yield 0 -def get_all_members(obj: Dict[str, Any], cluster: Cluster, - group: Optional[int], role: str = 'leader') -> Iterator[Member]: +def get_all_members(cluster: Cluster, group: Optional[int], role: str = 'leader') -> Iterator[Member]: """Get all cluster members that have the given *role*. - :param obj: the Patroni configuration. :param cluster: the Patroni cluster. :param group: filter which Citus group we should get members from. If ``None`` get from all groups. :param role: role to filter members. Can be one among: @@ -470,7 +485,7 @@ def get_all_members(obj: Dict[str, Any], cluster: Cluster, :yields: members that have the given *role*. """ clusters = {0: cluster} - if obj.get('citus') and group is None: + if is_citus_cluster() and group is None: clusters.update(cluster.workers) if role in ('leader', 'master', 'primary', 'standby-leader'): # In the DCS the members' role can be one among: ``primary``, ``master``, ``replica`` or ``standby_leader``. @@ -492,11 +507,10 @@ def get_all_members(obj: Dict[str, Any], cluster: Cluster, yield m -def get_any_member(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], +def get_any_member(cluster: Cluster, group: Optional[int], role: Optional[str] = None, member: Optional[str] = None) -> Optional[Member]: """Get the first found cluster member that has the given *role*. - :param obj: the Patroni configuration. :param cluster: the Patroni cluster. :param group: filter which Citus group we should get members from. If ``None`` get from all groups. :param role: role to filter members. See :func:`get_all_members` for available options. @@ -514,7 +528,7 @@ def get_any_member(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], elif role is None: role = 'leader' - for m in get_all_members(obj, cluster, group, role): + for m in get_all_members(cluster, group, role): if member is None or m.name == member: return m @@ -535,7 +549,7 @@ def get_all_members_leader_first(cluster: Cluster) -> Iterator[Member]: yield member -def get_cursor(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], connect_parameters: Dict[str, Any], +def get_cursor(cluster: Cluster, group: Optional[int], connect_parameters: Dict[str, Any], role: Optional[str] = None, member_name: Optional[str] = None) -> Union['cursor', 'Cursor[Any]', None]: """Get a cursor object to execute queries against a member that has the given *role* or *member_name*. @@ -544,7 +558,6 @@ def get_cursor(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], conn * ``fallback_application_name``: as ``Patroni ctl``; * ``connect_timeout``: as ``5``. - :param obj: the Patroni configuration. :param cluster: the Patroni cluster. :param group: filter which Citus group we should get members to create a cursor against. If ``None`` consider members from all groups. @@ -559,7 +572,7 @@ def get_cursor(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], conn * A :class:`psycopg2.extensions.cursor` if using :mod:`psycopg2`; * ``None`` if not able to get a cursor that attendees *role* and *member_name*. """ - member = get_any_member(obj, cluster, group, role=role, member=member_name) + member = get_any_member(cluster, group, role=role, member=member_name) if member is None: return None @@ -594,7 +607,7 @@ def get_cursor(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], conn return None -def get_members(obj: Dict[str, Any], cluster: Cluster, cluster_name: str, member_names: List[str], role: str, +def get_members(cluster: Cluster, cluster_name: str, member_names: List[str], role: str, force: bool, action: str, ask_confirmation: bool = True, group: Optional[int] = None) -> List[Member]: """Get the list of members based on the given filters. @@ -618,7 +631,6 @@ def get_members(obj: Dict[str, Any], cluster: Cluster, cluster_name: str, member ``ask_confirmation=False``, and later call :func:`confirm_members_action` manually in the caller method. That way the workflow won't look broken to the user that is interacting with ``patronictl``. - :param obj: Patroni configuration. :param cluster: Patroni cluster. :param cluster_name: name of the Patroni cluster. :param member_names: used to filter which members should take the *action* based on their names. Each item is the @@ -647,13 +659,13 @@ def get_members(obj: Dict[str, Any], cluster: Cluster, cluster_name: str, member * Cluster does not have members that match the given *member_names*; or * No member with given *role* is found among the specified *member_names*. """ - members = list(get_all_members(obj, cluster, group, role)) + members = list(get_all_members(cluster, group, role)) candidates = {m.name for m in members} if not force or role: if not member_names and not candidates: raise PatroniCtlException('{0} cluster doesn\'t have any members'.format(cluster_name)) - output_members(obj, cluster, cluster_name, group=group) + output_members(cluster, cluster_name, group=group) if member_names: member_names = list(set(member_names) & candidates) @@ -713,9 +725,7 @@ def confirm_members_action(members: List[Member], force: bool, action: str, @click.option('--member', '-m', help='Generate a dsn for this member', type=str) @arg_cluster_name @option_citus_group -@click.pass_obj -def dsn(obj: Dict[str, Any], cluster_name: str, group: Optional[int], - role: Optional[str], member: Optional[str]) -> None: +def dsn(cluster_name: str, group: Optional[int], role: Optional[str], member: Optional[str]) -> None: """Process ``dsn`` command of ``patronictl`` utility. Get DSN to connect to *member*. @@ -723,7 +733,6 @@ def dsn(obj: Dict[str, Any], cluster_name: str, group: Optional[int], .. note:: If no *role* nor *member* is given assume *role* as ``leader``. - :param obj: Patroni configuration. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should get members to get DSN from. Refer to the module note for more details. @@ -736,8 +745,8 @@ def dsn(obj: Dict[str, Any], cluster_name: str, group: Optional[int], * both *role* and *member* are provided; or * No member matches requested *member* or *role*. """ - cluster = get_dcs(obj, cluster_name, group).get_cluster() - m = get_any_member(obj, cluster, group, role=role, member=member) + cluster = get_dcs(cluster_name, group).get_cluster() + m = get_any_member(cluster, group, role=role, member=member) if m is None: raise PatroniCtlException('Can not find a suitable member') @@ -759,9 +768,7 @@ def dsn(obj: Dict[str, Any], cluster_name: str, group: Optional[int], @click.option('--delimiter', help='The column delimiter', default='\t') @click.option('--command', '-c', help='The SQL commands to execute') @click.option('-d', '--dbname', help='database name to connect to', type=str) -@click.pass_obj def query( - obj: Dict[str, Any], cluster_name: str, group: Optional[int], role: Optional[str], @@ -780,7 +787,6 @@ def query( Perform a Postgres query in a Patroni node. - :param obj: Patroni configuration. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should get members from to perform the query. Refer to the module note for more details. @@ -820,24 +826,22 @@ def query( if dbname: connect_parameters['dbname'] = dbname - dcs = get_dcs(obj, cluster_name, group) + dcs = get_dcs(cluster_name, group) cluster = cursor = None for _ in watching(w, watch, clear=False): if cluster is None: cluster = dcs.get_cluster() -# cursor = get_cursor(obj, cluster, group, connect_parameters, role=role, member=member) - output, header = query_member(obj, cluster, group, cursor, member, role, sql, connect_parameters) + output, header = query_member(cluster, group, cursor, member, role, sql, connect_parameters) print_output(header, output, fmt=fmt, delimiter=delimiter) -def query_member(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], - cursor: Union['cursor', 'Cursor[Any]', None], member: Optional[str], role: Optional[str], - command: str, connect_parameters: Dict[str, Any]) -> Tuple[List[List[Any]], Optional[List[Any]]]: +def query_member(cluster: Cluster, group: Optional[int], cursor: Union['cursor', 'Cursor[Any]', None], + member: Optional[str], role: Optional[str], command: str, + connect_parameters: Dict[str, Any]) -> Tuple[List[List[Any]], Optional[List[Any]]]: """Execute SQL *command* against a member. - :param obj: Patroni configuration. :param cluster: the Patroni cluster. :param group: filter which Citus group we should get members from to perform the query. Refer to the module note for more details. @@ -866,7 +870,7 @@ def query_member(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], from . import psycopg try: if cursor is None: - cursor = get_cursor(obj, cluster, group, connect_parameters, role=role, member_name=member) + cursor = get_cursor(cluster, group, connect_parameters, role=role, member_name=member) if cursor is None: if member is not None: @@ -893,13 +897,11 @@ def query_member(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], @click.argument('cluster_name') @option_citus_group @option_format -@click.pass_obj -def remove(obj: Dict[str, Any], cluster_name: str, group: Optional[int], fmt: str) -> None: +def remove(cluster_name: str, group: Optional[int], fmt: str) -> None: """Process ``remove`` command of ``patronictl`` utility. Remove cluster *cluster_name* from the DCS. - :param obj: Patroni configuration. :param cluster_name: name of the cluster which information will be wiped out of the DCS. :param group: which Citus group should have its information wiped out of the DCS. Refer to the module note for more details. @@ -913,12 +915,12 @@ def remove(obj: Dict[str, Any], cluster_name: str, group: Optional[int], fmt: st * use did not type the correct leader name when requesting removal of a healthy cluster. """ - dcs = get_dcs(obj, cluster_name, group) + dcs = get_dcs(cluster_name, group) cluster = dcs.get_cluster() - if obj.get('citus') and group is None: + if is_citus_cluster() and group is None: raise PatroniCtlException('For Citus clusters the --group must me specified') - output_members(obj, cluster, cluster_name, fmt=fmt) + output_members(cluster, cluster_name, fmt=fmt) confirm = click.prompt('Please confirm the cluster name to remove', type=str) if confirm != cluster_name: @@ -1003,24 +1005,21 @@ def parse_scheduled(scheduled: Optional[str]) -> Optional[datetime.datetime]: @option_citus_group @click.option('--role', '-r', help='Reload only members with this role', type=role_choice, default='any') @option_force -@click.pass_obj -def reload(obj: Dict[str, Any], cluster_name: str, member_names: List[str], - group: Optional[int], force: bool, role: str) -> None: +def reload(cluster_name: str, member_names: List[str], group: Optional[int], force: bool, role: str) -> None: """Process ``reload`` command of ``patronictl`` utility. Reload configuration of cluster members based on given filters. - :param obj: Patroni configuration. :param cluster_name: name of the Patroni cluster. :param member_names: name of the members which configuration should be reloaded. :param group: filter which Citus group we should reload members. Refer to the module note for more details. :param force: perform the reload without asking for confirmations. :param role: role to filter members. See :func:`get_all_members` for available options. """ - dcs = get_dcs(obj, cluster_name, group) + dcs = get_dcs(cluster_name, group) cluster = dcs.get_cluster() - members = get_members(obj, cluster, cluster_name, member_names, role, force, 'reload', group=group) + members = get_members(cluster, cluster_name, member_names, role, force, 'reload', group=group) for member in members: r = request_patroni(member, 'post', 'reload') @@ -1050,15 +1049,13 @@ def reload(obj: Dict[str, Any], cluster_name: str, member_names: List[str], @click.option('--pending', help='Restart if pending', is_flag=True) @click.option('--timeout', help='Return error and fail over if necessary when restarting takes longer than this.') @option_force -@click.pass_obj -def restart(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member_names: List[str], +def restart(cluster_name: str, group: Optional[int], member_names: List[str], force: bool, role: str, p_any: bool, scheduled: Optional[str], version: Optional[str], pending: bool, timeout: Optional[str]) -> None: """Process ``restart`` command of ``patronictl`` utility. Restart Postgres on cluster members based on given filters. - :param obj: Patroni configuration. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should restart members. Refer to the module note for more details. :param member_names: name of the members that should be restarted. @@ -1076,9 +1073,9 @@ def restart(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member * *version* could not be parsed; or * a restart is attempted against a cluster that is in maintenance mode. """ - cluster = get_dcs(obj, cluster_name, group).get_cluster() + cluster = get_dcs(cluster_name, group).get_cluster() - members = get_members(obj, cluster, cluster_name, member_names, role, force, 'restart', False, group=group) + members = get_members(cluster, cluster_name, member_names, role, force, 'restart', False, group=group) if scheduled is None and not force: next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M') scheduled = click.prompt('When should the restart take place (e.g. ' + next_hour + ') ', @@ -1140,9 +1137,7 @@ def restart(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member @click.argument('member_names', nargs=-1) @option_force @click.option('--wait', help='Wait until reinitialization completes', is_flag=True) -@click.pass_obj -def reinit(obj: Dict[str, Any], cluster_name: str, group: Optional[int], - member_names: List[str], force: bool, wait: bool) -> None: +def reinit(cluster_name: str, group: Optional[int], member_names: List[str], force: bool, wait: bool) -> None: """Process ``reinit`` command of ``patronictl`` utility. Reinitialize cluster members based on given filters. @@ -1150,15 +1145,14 @@ def reinit(obj: Dict[str, Any], cluster_name: str, group: Optional[int], .. note:: Only reinitialize replica members, not a leader. - :param obj: Patroni configuration. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should reinit members. Refer to the module note for more details. :param member_names: name of the members that should be reinitialized. :param force: perform the restart without asking for confirmations. :param wait: wait for the operation to complete. """ - cluster = get_dcs(obj, cluster_name, group).get_cluster() - members = get_members(obj, cluster, cluster_name, member_names, 'replica', force, 'reinitialize', group=group) + cluster = get_dcs(cluster_name, group).get_cluster() + members = get_members(cluster, cluster_name, member_names, 'replica', force, 'reinitialize', group=group) wait_on_members: List[Member] = [] for member in members: @@ -1189,8 +1183,8 @@ def reinit(obj: Dict[str, Any], cluster_name: str, group: Optional[int], wait_on_members.remove(member) -def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: str, - group: Optional[int], leader: Optional[str], candidate: Optional[str], +def _do_failover_or_switchover(action: str, cluster_name: str, group: Optional[int], + leader: Optional[str], candidate: Optional[str], force: bool, scheduled: Optional[str] = None) -> None: """Perform a failover or a switchover operation in the cluster. @@ -1200,7 +1194,6 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s .. note:: If not able to perform the operation through the REST API, write directly to the DCS as a fall back. - :param obj: Patroni configuration. :param action: action to be taken -- ``failover`` or ``switchover``. :param cluster_name: name of the Patroni cluster. :param group: filter Citus group within we should perform a failover or switchover. If ``None``, user will be @@ -1222,17 +1215,17 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s * trying to schedule a switchover in a cluster that is in maintenance mode; or * user aborts the operation. """ - dcs = get_dcs(obj, cluster_name, group) + dcs = get_dcs(cluster_name, group) cluster = dcs.get_cluster() click.echo('Current cluster topology') - output_members(obj, cluster, cluster_name, group=group) + output_members(cluster, cluster_name, group=group) - if obj.get('citus') and group is None: + if is_citus_cluster() and group is None: if force: raise PatroniCtlException('For Citus clusters the --group must me specified') else: group = click.prompt('Citus group', type=int) - dcs = get_dcs(obj, cluster_name, group) + dcs = get_dcs(cluster_name, group) cluster = dcs.get_cluster() global_config = get_global_config(cluster) @@ -1342,7 +1335,7 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s click.echo('{0} Could not {1} using Patroni api, falling back to DCS'.format(timestamp(), action)) dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at) - output_members(obj, cluster, cluster_name, group=group) + output_members(cluster, cluster_name, group=group) @ctl.command('failover', help='Failover to a replica') @@ -1351,8 +1344,7 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s @click.option('--leader', '--primary', '--master', 'leader', help='The name of the current leader', default=None) @click.option('--candidate', help='The name of the candidate', default=None) @option_force -@click.pass_obj -def failover(obj: Dict[str, Any], cluster_name: str, group: Optional[int], +def failover(cluster_name: str, group: Optional[int], leader: Optional[str], candidate: Optional[str], force: bool) -> None: """Process ``failover`` command of ``patronictl`` utility. @@ -1366,7 +1358,6 @@ def failover(obj: Dict[str, Any], cluster_name: str, group: Optional[int], .. seealso:: Refer to :func:`_do_failover_or_switchover` for details. - :param obj: Patroni configuration. :param cluster_name: name of the Patroni cluster. :param group: filter Citus group within we should perform a failover or switchover. If ``None``, user will be prompted for filling it -- unless *force* is ``True``, in which case an exception is raised by @@ -1381,7 +1372,7 @@ def failover(obj: Dict[str, Any], cluster_name: str, group: Optional[int], click.echo(click.style( 'Supplying a leader name using this command is deprecated and will be removed in a future version of' ' Patroni, change your scripts to use `switchover` instead.\nExecuting switchover!', fg='red')) - _do_failover_or_switchover(obj, action, cluster_name, group, leader, candidate, force) + _do_failover_or_switchover(action, cluster_name, group, leader, candidate, force) @ctl.command('switchover', help='Switchover to a replica') @@ -1392,9 +1383,8 @@ def failover(obj: Dict[str, Any], cluster_name: str, group: Optional[int], @click.option('--scheduled', help='Timestamp of a scheduled switchover in unambiguous format (e.g. ISO 8601)', default=None) @option_force -@click.pass_obj -def switchover(obj: Dict[str, Any], cluster_name: str, group: Optional[int], - leader: Optional[str], candidate: Optional[str], force: bool, scheduled: Optional[str]) -> None: +def switchover(cluster_name: str, group: Optional[int], leader: Optional[str], + candidate: Optional[str], force: bool, scheduled: Optional[str]) -> None: """Process ``switchover`` command of ``patronictl`` utility. Perform a switchover operation in the cluster. @@ -1402,7 +1392,6 @@ def switchover(obj: Dict[str, Any], cluster_name: str, group: Optional[int], .. seealso:: Refer to :func:`_do_failover_or_switchover` for details. - :param obj: Patroni configuration. :param cluster_name: name of the Patroni cluster. :param group: filter Citus group within we should perform a switchover. If ``None``, user will be prompted for filling it -- unless *force* is ``True``, in which case an exception is raised by @@ -1412,7 +1401,7 @@ def switchover(obj: Dict[str, Any], cluster_name: str, group: Optional[int], :param force: perform the switchover without asking for confirmations. :param scheduled: timestamp when the switchover should be scheduled to occur. If ``now`` perform immediately. """ - _do_failover_or_switchover(obj, 'switchover', cluster_name, group, leader, candidate, force, scheduled) + _do_failover_or_switchover('switchover', cluster_name, group, leader, candidate, force, scheduled) def generate_topology(level: int, member: Dict[str, Any], @@ -1514,8 +1503,8 @@ def get_cluster_service_info(cluster: Dict[str, Any]) -> List[str]: return service_info -def output_members(obj: Dict[str, Any], cluster: Cluster, name: str, - extended: bool = False, fmt: str = 'pretty', group: Optional[int] = None) -> None: +def output_members(cluster: Cluster, name: str, extended: bool = False, + fmt: str = 'pretty', group: Optional[int] = None) -> None: """Print information about the Patroni cluster and its members. Information is printed to console through :func:`print_output`, and contains: @@ -1540,7 +1529,6 @@ def output_members(obj: Dict[str, Any], cluster: Cluster, name: str, The 3 extended columns are always included if *extended*, even if the member has no value for a given column. If not *extended*, these columns may still be shown if any of the members has any information for them. - :param obj: Patroni configuration. :param cluster: Patroni cluster. :param name: name of the Patroni cluster. :param extended: if extended information (pending restarts, scheduled restarts, node tags) should be printed, if @@ -1558,8 +1546,7 @@ def output_members(obj: Dict[str, Any], cluster: Cluster, name: str, clusters = {group or 0: cluster_as_json(cluster)} - is_citus_cluster = obj.get('citus') - if is_citus_cluster: + if is_citus_cluster(): columns.insert(1, 'Group') if group is None: clusters.update({g: cluster_as_json(c) for g, c in cluster.workers.items()}) @@ -1597,10 +1584,12 @@ def output_members(obj: Dict[str, Any], cluster: Cluster, name: str, rows.append([member.get(n.lower().replace(' ', '_'), '') for n in columns]) - title = 'Citus cluster' if is_citus_cluster else 'Cluster' - title_details = f' ({initialize})' - if is_citus_cluster: + if is_citus_cluster(): + title = 'Citus cluster' title_details = '' if group is None else f' (group: {group}, {initialize})' + else: + title = 'Cluster' + title_details = f' ({initialize})' title = f' {title}: {name}{title_details} ' print_output(columns, rows, {'Group': 'r', 'Lag in MB': 'r', 'TL': 'r'}, fmt, title) @@ -1611,7 +1600,7 @@ def output_members(obj: Dict[str, Any], cluster: Cluster, name: str, for g, c in sorted(clusters.items()): service_info = get_cluster_service_info(c) if service_info: - if is_citus_cluster and group is None: + if is_citus_cluster() and group is None: click.echo('Citus group: {0}'.format(g)) click.echo(' ' + '\n '.join(service_info)) @@ -1624,16 +1613,14 @@ def output_members(obj: Dict[str, Any], cluster: Cluster, name: str, @option_format @option_watch @option_watchrefresh -@click.pass_obj -def members(obj: Dict[str, Any], cluster_names: List[str], group: Optional[int], - fmt: str, watch: Optional[int], w: bool, extended: bool, ts: bool) -> None: +def members(cluster_names: List[str], group: Optional[int], fmt: str, + watch: Optional[int], w: bool, extended: bool, ts: bool) -> None: """Process ``list`` command of ``patronictl`` utility. Print information about the Patroni cluster through :func:`output_members`. - :param obj: Patroni configuration. :param cluster_names: name of clusters that should be printed. If ``None`` consider only the cluster present in - ``scope`` key of *obj*. + ``scope`` key of the configuration. :param group: filter which Citus group we should get members from. Refer to the module note for more details. :param fmt: the output table printing format. See :func:`print_output` for available options. :param watch: if given print output every *watch* seconds. @@ -1642,9 +1629,10 @@ def members(obj: Dict[str, Any], cluster_names: List[str], group: Optional[int], more details. :param ts: if timestamp should be included in the output. """ + config = _get_configuration() if not cluster_names: - if 'scope' in obj: - cluster_names = [obj['scope']] + if 'scope' in config: + cluster_names = [config['scope']] if not cluster_names: return logging.warning('Listing members: No cluster names were provided') @@ -1653,10 +1641,10 @@ def members(obj: Dict[str, Any], cluster_names: List[str], group: Optional[int], click.echo(timestamp(0)) for cluster_name in cluster_names: - dcs = get_dcs(obj, cluster_name, group) + dcs = get_dcs(cluster_name, group) cluster = dcs.get_cluster() - output_members(obj, cluster, cluster_name, extended, fmt, group) + output_members(cluster, cluster_name, extended, fmt, group) @ctl.command('topology', help='Prints ASCII topology for given cluster') @@ -1698,14 +1686,12 @@ def timestamp(precision: int = 6) -> str: @click.argument('target', type=click.Choice(['restart', 'switchover'])) @click.option('--role', '-r', help='Flush only members with this role', type=role_choice, default='any') @option_force -@click.pass_obj -def flush(obj: Dict[str, Any], cluster_name: str, group: Optional[int], +def flush(cluster_name: str, group: Optional[int], member_names: List[str], force: bool, role: str, target: str) -> None: """Process ``flush`` command of ``patronictl`` utility. Discard scheduled restart or switchover events. - :param obj: Patroni configuration. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should flush an event. Refer to the module note for more details. :param member_names: name of the members which events should be flushed. @@ -1713,11 +1699,11 @@ def flush(obj: Dict[str, Any], cluster_name: str, group: Optional[int], :param role: role to filter members. See :func:`get_all_members` for available options. :param target: the event that should be flushed -- ``restart`` or ``switchover``. """ - dcs = get_dcs(obj, cluster_name, group) + dcs = get_dcs(cluster_name, group) cluster = dcs.get_cluster() if target == 'restart': - for member in get_members(obj, cluster, cluster_name, member_names, role, force, 'flush', group=group): + for member in get_members(cluster, cluster_name, member_names, role, force, 'flush', group=group): if member.data.get('scheduled_restart'): r = request_patroni(member, 'delete', 'restart') check_response(r, member.name, 'flush scheduled restart') @@ -1775,10 +1761,9 @@ def wait_until_pause_is_applied(dcs: AbstractDCS, paused: bool, old_cluster: Clu return click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed')) -def toggle_pause(config: Dict[str, Any], cluster_name: str, group: Optional[int], paused: bool, wait: bool) -> None: +def toggle_pause(cluster_name: str, group: Optional[int], paused: bool, wait: bool) -> None: """Toggle the ``pause`` state in the cluster members. - :param config: Patroni configuration. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should toggle the pause state of. Refer to the module note for more details. @@ -1790,7 +1775,7 @@ def toggle_pause(config: Dict[str, Any], cluster_name: str, group: Optional[int] * ``pause`` state is already *paused*; or * cluster contains no accessible members. """ - dcs = get_dcs(config, cluster_name, group) + dcs = get_dcs(cluster_name, group) cluster = dcs.get_cluster() if get_global_config(cluster).is_paused == paused: raise PatroniCtlException('Cluster is {0} paused'.format(paused and 'already' or 'not')) @@ -1819,37 +1804,33 @@ def toggle_pause(config: Dict[str, Any], cluster_name: str, group: Optional[int] @ctl.command('pause', help='Disable auto failover') @arg_cluster_name @option_default_citus_group -@click.pass_obj @click.option('--wait', help='Wait until pause is applied on all nodes', is_flag=True) -def pause(obj: Dict[str, Any], cluster_name: str, group: Optional[int], wait: bool) -> None: +def pause(cluster_name: str, group: Optional[int], wait: bool) -> None: """Process ``pause`` command of ``patronictl`` utility. Put the cluster in maintenance mode. - :param obj: Patroni configuration. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should pause. Refer to the module note for more details. :param wait: ``True`` if it should block until the operation is finished or ``false`` for returning immediately. """ - return toggle_pause(obj, cluster_name, group, True, wait) + return toggle_pause(cluster_name, group, True, wait) @ctl.command('resume', help='Resume auto failover') @arg_cluster_name @option_default_citus_group @click.option('--wait', help='Wait until pause is cleared on all nodes', is_flag=True) -@click.pass_obj -def resume(obj: Dict[str, Any], cluster_name: str, group: Optional[int], wait: bool) -> None: +def resume(cluster_name: str, group: Optional[int], wait: bool) -> None: """Process ``unpause`` command of ``patronictl`` utility. Put the cluster out of maintenance mode. - :param obj: Patroni configuration. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should unpause. Refer to the module note for more details. :param wait: ``True`` if it should block until the operation is finished or ``false`` for returning immediately. """ - return toggle_pause(obj, cluster_name, group, False, wait) + return toggle_pause(cluster_name, group, False, wait) @contextmanager @@ -2081,15 +2062,12 @@ def invoke_editor(before_editing: str, cluster_name: str) -> Tuple[str, Dict[str @click.option('--replace', 'replace_filename', help='Apply configuration from file, replacing existing configuration.' ' Use - for stdin.') @option_force -@click.pass_obj -def edit_config(obj: Dict[str, Any], cluster_name: str, group: Optional[int], - force: bool, quiet: bool, kvpairs: List[str], pgkvpairs: List[str], - apply_filename: Optional[str], replace_filename: Optional[str]) -> None: +def edit_config(cluster_name: str, group: Optional[int], force: bool, quiet: bool, kvpairs: List[str], + pgkvpairs: List[str], apply_filename: Optional[str], replace_filename: Optional[str]) -> None: """Process ``edit-config`` command of ``patronictl`` utility. Update or replace Patroni configuration in the DCS. - :param obj: Patroni configuration. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group configuration we should edit. Refer to the module note for more details. :param force: if ``True`` apply config changes without asking for confirmations. @@ -2106,7 +2084,7 @@ def edit_config(obj: Dict[str, Any], cluster_name: str, group: Optional[int], * Configuration is absent from DCS; or * Detected a concurrent modification of the configuration in the DCS. """ - dcs = get_dcs(obj, cluster_name, group) + dcs = get_dcs(cluster_name, group) cluster = dcs.get_cluster() if not cluster.config: @@ -2152,17 +2130,15 @@ def edit_config(obj: Dict[str, Any], cluster_name: str, group: Optional[int], @ctl.command('show-config', help="Show cluster configuration") @arg_cluster_name @option_default_citus_group -@click.pass_obj -def show_config(obj: Dict[str, Any], cluster_name: str, group: Optional[int]) -> None: +def show_config(cluster_name: str, group: Optional[int]) -> None: """Process ``show-config`` command of ``patronictl`` utility. Show Patroni configuration stored in the DCS. - :param obj: Patroni configuration. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group configuration we should show. Refer to the module note for more details. """ - cluster = get_dcs(obj, cluster_name, group).get_cluster() + cluster = get_dcs(cluster_name, group).get_cluster() if cluster.config: click.echo(format_config_for_editing(cluster.config.data)) @@ -2171,8 +2147,7 @@ def show_config(obj: Dict[str, Any], cluster_name: str, group: Optional[int]) -> @click.argument('cluster_name', required=False) @click.argument('member_names', nargs=-1) @option_citus_group -@click.pass_obj -def version(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member_names: List[str]) -> None: +def version(cluster_name: str, group: Optional[int], member_names: List[str]) -> None: """Process ``version`` command of ``patronictl`` utility. Show version of: @@ -2180,7 +2155,6 @@ def version(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member * ``patroni`` on all members of the cluster; * ``PostgreSQL`` on all members of the cluster. - :param obj: Patroni configuration. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should get members from. Refer to the module note for more details. :param member_names: filter which members we should get version information from. @@ -2191,8 +2165,8 @@ def version(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member return click.echo("") - cluster = get_dcs(obj, cluster_name, group).get_cluster() - for m in get_all_members(obj, cluster, group, 'any'): + cluster = get_dcs(cluster_name, group).get_cluster() + for m in get_all_members(cluster, group, 'any'): if m.api_url: if not member_names or m.name in member_names: try: @@ -2210,8 +2184,7 @@ def version(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member @arg_cluster_name @option_default_citus_group @option_format -@click.pass_obj -def history(obj: Dict[str, Any], cluster_name: str, group: Optional[int], fmt: str) -> None: +def history(cluster_name: str, group: Optional[int], fmt: str) -> None: """Process ``history`` command of ``patronictl`` utility. Show the history of failover/switchover events in the cluster. @@ -2223,12 +2196,11 @@ def history(obj: Dict[str, Any], cluster_name: str, group: Optional[int], fmt: s * ``Timestamp``: timestamp when the event occurred; * ``New Leader``: the Postgres node that was promoted during the event. - :param obj: Patroni configuration. :param cluster_name: name of the Patroni cluster. :param group: filter which Citus group we should get events from. Refer to the module note for more details. :param fmt: the output table printing format. See :func:`print_output` for available options. """ - cluster = get_dcs(obj, cluster_name, group).get_cluster() + cluster = get_dcs(cluster_name, group).get_cluster() cluster_history = cluster.history.lines if cluster.history else [] history: List[List[Any]] = list(map(list, cluster_history)) table_header_row = ['TL', 'LSN', 'Reason', 'Timestamp', 'New Leader'] diff --git a/tests/test_ctl.py b/tests/test_ctl.py index 85691812..bacd2d60 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -1,3 +1,4 @@ +import click import etcd import mock import os @@ -9,7 +10,7 @@ from mock import patch, Mock, PropertyMock from patroni.ctl import ctl, load_config, output_members, get_dcs, parse_dcs, \ get_all_members, get_any_member, get_cursor, query_member, PatroniCtlException, apply_config_changes, \ format_config_for_editing, show_diff, invoke_editor, format_pg_version, CONFIG_FILE_PATH, PatronictlPrettyTable -from patroni.dcs.etcd import AbstractEtcdClientWithFailover, Cluster, Failover +from patroni.dcs import Cluster, Failover from patroni.psycopg import OperationalError from patroni.utils import tzutc from prettytable import PrettyTable, ALL @@ -21,26 +22,26 @@ from .test_ha import get_cluster_initialized_without_leader, get_cluster_initial get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader, get_cluster, Member -DEFAULT_CONFIG = { - 'scope': 'alpha', - 'restapi': {'listen': '::', 'certfile': 'a'}, - 'ctl': {'certfile': 'a'}, - 'etcd': {'host': 'localhost:2379'}, - 'citus': {'database': 'citus', 'group': 0}, - 'postgresql': {'data_dir': '.', 'pgpass': './pgpass', 'parameters': {}, 'retry_timeout': 5} -} +def get_default_config(*args): + return { + 'scope': 'alpha', + 'restapi': {'listen': '::', 'certfile': 'a'}, + 'ctl': {'certfile': 'a'}, + 'etcd': {'host': 'localhost:2379', 'retry_timeout': 10, 'ttl': 30}, + 'citus': {'database': 'citus', 'group': 0}, + 'postgresql': {'data_dir': '.', 'pgpass': './pgpass', 'parameters': {}, 'retry_timeout': 5} + } -@patch('patroni.ctl.load_config', Mock(return_value=DEFAULT_CONFIG)) +@patch.object(PoolManager, 'request', Mock(return_value=MockResponse())) +@patch('patroni.ctl.load_config', get_default_config) +@patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) class TestCtl(unittest.TestCase): TEST_ROLES = ('master', 'primary', 'leader') @patch('socket.getaddrinfo', socket_getaddrinfo) - @patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379'])) def setUp(self): self.runner = CliRunner() - self.e = get_dcs({'etcd': {'ttl': 30, 'host': 'ok:2379', 'retry_timeout': 10}, - 'citus': {'group': 0}}, 'foo', None) @patch('patroni.ctl.logging.debug') def test_load_config(self, mock_logger_debug): @@ -66,29 +67,31 @@ class TestCtl(unittest.TestCase): @patch('patroni.psycopg.connect', psycopg_connect) def test_get_cursor(self): - for role in self.TEST_ROLES: - self.assertIsNone(get_cursor({}, get_cluster_initialized_without_leader(), None, {}, role=role)) - self.assertIsNotNone(get_cursor({}, get_cluster_initialized_with_leader(), None, {}, role=role)) + with click.Context(click.Command('query')) as ctx: + ctx.obj = {'__config': {}} + for role in self.TEST_ROLES: + self.assertIsNone(get_cursor(get_cluster_initialized_without_leader(), None, {}, role=role)) + self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), None, {}, role=role)) - # MockCursor returns pg_is_in_recovery as false - self.assertIsNone(get_cursor({}, get_cluster_initialized_with_leader(), None, {}, role='replica')) + # MockCursor returns pg_is_in_recovery as false + self.assertIsNone(get_cursor(get_cluster_initialized_with_leader(), None, {}, role='replica')) - self.assertIsNotNone(get_cursor({}, get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, role='any')) + self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, role='any')) - # Mutually exclusive options - with self.assertRaises(PatroniCtlException) as e: - get_cursor({}, get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, member_name='other', - role='replica') + # Mutually exclusive options + with self.assertRaises(PatroniCtlException) as e: + get_cursor(get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, member_name='other', + role='replica') - self.assertEqual(str(e.exception), '--role and --member are mutually exclusive options') + self.assertEqual(str(e.exception), '--role and --member are mutually exclusive options') - # Invalid member provided - self.assertIsNone(get_cursor({}, get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, - member_name='invalid')) + # Invalid member provided + self.assertIsNone(get_cursor(get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, + member_name='invalid')) - # Valid member provided - self.assertIsNotNone(get_cursor({}, get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, - member_name='other')) + # Valid member provided + self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, + member_name='other')) def test_parse_dcs(self): assert parse_dcs(None) is None @@ -102,23 +105,20 @@ class TestCtl(unittest.TestCase): self.assertRaises(PatroniCtlException, parse_dcs, 'invalid://test') def test_output_members(self): - scheduled_at = datetime.now(tzutc) + timedelta(seconds=600) - cluster = get_cluster_initialized_with_leader(Failover(1, 'foo', 'bar', scheduled_at)) - del cluster.members[1].data['conn_url'] - for fmt in ('pretty', 'json', 'yaml', 'topology'): - self.assertIsNone(output_members({}, cluster, name='abc', fmt=fmt)) + with click.Context(click.Command('list')) as ctx: + ctx.obj = {'__config': {}} + scheduled_at = datetime.now(tzutc) + timedelta(seconds=600) + cluster = get_cluster_initialized_with_leader(Failover(1, 'foo', 'bar', scheduled_at)) + del cluster.members[1].data['conn_url'] + for fmt in ('pretty', 'json', 'yaml', 'topology'): + self.assertIsNone(output_members(cluster, name='abc', fmt=fmt)) - with patch('click.echo') as mock_echo: - self.assertIsNone(output_members({}, cluster, name='abc', fmt='tsv')) - self.assertEqual(mock_echo.call_args[0][0], 'abc\tother\t\tReplica\trunning\t\tunknown') - - @patch('patroni.ctl.get_dcs') - @patch.object(PoolManager, 'request', Mock(return_value=MockResponse())) - def test_switchover(self, mock_get_dcs): - mock_get_dcs.return_value = self.e - mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader - mock_get_dcs.return_value.set_failover_value = Mock() + with patch('click.echo') as mock_echo: + self.assertIsNone(output_members(cluster, name='abc', fmt='tsv')) + self.assertEqual(mock_echo.call_args[0][0], 'abc\tother\t\tReplica\trunning\t\tunknown') + @patch('patroni.dcs.AbstractDCS.set_failover_value', Mock()) + def test_switchover(self): # Confirm result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') self.assertEqual(result.exit_code, 0) @@ -180,12 +180,12 @@ class TestCtl(unittest.TestCase): self.assertIn('Member dummy is not the leader of cluster dummy', result.output) # Errors while sending Patroni REST API request - with patch.object(PoolManager, 'request', Mock(side_effect=Exception)): + with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)): result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n2300-01-01T12:23:00\ny') self.assertIn('falling back to DCS', result.output) - with patch.object(PoolManager, 'request') as mock_api_request: + with patch('patroni.ctl.request_patroni') as mock_api_request: mock_api_request.return_value.status = 500 result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') self.assertIn('Switchover failed', result.output) @@ -196,64 +196,58 @@ class TestCtl(unittest.TestCase): self.assertIn('Switchover failed', result.output) # No members available - mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') - self.assertEqual(result.exit_code, 1) - self.assertIn('No candidates found to switchover to', result.output) + with patch('patroni.dcs.AbstractDCS.get_cluster', + Mock(return_value=get_cluster_initialized_with_only_leader())): + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') + self.assertEqual(result.exit_code, 1) + self.assertIn('No candidates found to switchover to', result.output) # No leader available - mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') - self.assertEqual(result.exit_code, 1) - self.assertIn('This cluster has no leader', result.output) + with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_without_leader())): + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') + self.assertEqual(result.exit_code, 1) + self.assertIn('This cluster has no leader', result.output) # Citus cluster, no group number specified result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force'], input='\n') self.assertEqual(result.exit_code, 1) self.assertIn('For Citus clusters the --group must me specified', result.output) - @patch('patroni.ctl.get_dcs') - @patch.object(PoolManager, 'request', Mock(return_value=MockResponse())) - @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse())) - def test_failover(self, mock_get_dcs): - mock_get_dcs.return_value.set_failover_value = Mock() - + @patch('patroni.dcs.AbstractDCS.set_failover_value', Mock()) + def test_failover(self): # No candidate specified - mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader result = self.runner.invoke(ctl, ['failover', 'dummy'], input='0\n') self.assertIn('Failover could be performed only to a specific candidate', result.output) - cluster = get_cluster_initialized_with_leader(sync=('leader', 'other')) - # Temp test to check a fallback to switchover if leader is specified with patch('patroni.ctl._do_failover_or_switchover') as failover_func_mock: result = self.runner.invoke(ctl, ['failover', '--leader', 'leader', 'dummy'], input='0\n') self.assertIn('Supplying a leader name using this command is deprecated', result.output) - failover_func_mock.assert_called_once_with( - DEFAULT_CONFIG, 'switchover', 'dummy', None, 'leader', None, False) + failover_func_mock.assert_called_once_with('switchover', 'dummy', None, 'leader', None, False) # Failover to an async member in sync mode (confirm) + cluster = get_cluster_initialized_with_leader(sync=('leader', 'other')) cluster.members.append(Member(0, 'async', 28, {'api_url': 'http://127.0.0.1:8012/patroni'})) cluster.config.data['synchronous_mode'] = True - mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster) - result = self.runner.invoke(ctl, ['failover', 'dummy', '--group', '0', '--candidate', 'async'], input='y\ny') - self.assertIn('Are you sure you want to failover to the asynchronous node async', result.output) + with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=cluster)): + result = self.runner.invoke(ctl, + ['failover', 'dummy', '--group', '0', '--candidate', 'async'], input='y\ny') + self.assertIn('Are you sure you want to failover to the asynchronous node async', result.output) # Failover to an async member in sync mode (abort) - mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster) result = self.runner.invoke(ctl, ['failover', 'dummy', '--group', '0', '--candidate', 'async'], input='N') self.assertEqual(result.exit_code, 1) @patch('patroni.dcs.dcs_modules', Mock(return_value=['patroni.dcs.dummy', 'patroni.dcs.etcd'])) def test_get_dcs(self): - self.assertRaises(PatroniCtlException, get_dcs, {'dummy': {}}, 'dummy', 0) + with click.Context(click.Command('list')) as ctx: + ctx.obj = {'__config': {'dummy': {}}} + self.assertRaises(PatroniCtlException, get_dcs, 'dummy', 0) @patch('patroni.psycopg.connect', psycopg_connect) @patch('patroni.ctl.query_member', Mock(return_value=([['mock column']], None))) - @patch('patroni.ctl.get_dcs') @patch.object(etcd.Client, 'read', etcd_read) - def test_query(self, mock_get_dcs): - mock_get_dcs.return_value = self.e + def test_query(self): # Mutually exclusive for role in self.TEST_ROLES: result = self.runner.invoke(ctl, ['query', 'alpha', '--member', 'abc', '--role', role]) @@ -286,31 +280,29 @@ class TestCtl(unittest.TestCase): def test_query_member(self): with patch('patroni.ctl.get_cursor', Mock(return_value=MockConnect().cursor())): for role in self.TEST_ROLES: - rows = query_member({}, None, None, None, None, role, 'SELECT pg_catalog.pg_is_in_recovery()', {}) + rows = query_member(None, None, None, None, role, 'SELECT pg_catalog.pg_is_in_recovery()', {}) self.assertTrue('False' in str(rows)) with patch.object(MockCursor, 'execute', Mock(side_effect=OperationalError('bla'))): - rows = query_member({}, None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {}) + rows = query_member(None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {}) with patch('patroni.ctl.get_cursor', Mock(return_value=None)): # No role nor member given -- generic message - rows = query_member({}, None, None, None, None, None, 'SELECT pg_catalog.pg_is_in_recovery()', {}) + rows = query_member(None, None, None, None, None, 'SELECT pg_catalog.pg_is_in_recovery()', {}) self.assertTrue('No connection is available' in str(rows)) # Member given -- message pointing to member - rows = query_member({}, None, None, None, 'foo', None, 'SELECT pg_catalog.pg_is_in_recovery()', {}) + rows = query_member(None, None, None, 'foo', None, 'SELECT pg_catalog.pg_is_in_recovery()', {}) self.assertTrue('No connection to member foo' in str(rows)) # Role given -- message pointing to role - rows = query_member({}, None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {}) + rows = query_member(None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {}) self.assertTrue('No connection to role replica' in str(rows)) with patch('patroni.ctl.get_cursor', Mock(side_effect=OperationalError('bla'))): - rows = query_member({}, None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {}) + rows = query_member(None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {}) - @patch('patroni.ctl.get_dcs') - def test_dsn(self, mock_get_dcs): - mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader + def test_dsn(self): result = self.runner.invoke(ctl, ['dsn', 'alpha']) assert 'host=127.0.0.1 port=5435' in result.output @@ -323,11 +315,8 @@ class TestCtl(unittest.TestCase): result = self.runner.invoke(ctl, ['dsn', 'alpha', '--member', 'dummy']) assert result.exit_code == 1 - @patch.object(PoolManager, 'request') - @patch('patroni.ctl.get_dcs') - def test_reload(self, mock_get_dcs, mock_post): - mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader - + @patch('patroni.ctl.request_patroni') + def test_reload(self, mock_post): result = self.runner.invoke(ctl, ['reload', 'alpha'], input='y') assert 'Failed: reload for member' in result.output @@ -339,10 +328,8 @@ class TestCtl(unittest.TestCase): result = self.runner.invoke(ctl, ['reload', 'alpha'], input='y') assert 'Reload request received for member' in result.output - @patch.object(PoolManager, 'request') - @patch('patroni.ctl.get_dcs') - def test_restart_reinit(self, mock_get_dcs, mock_post): - mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader + @patch('patroni.ctl.request_patroni') + def test_restart_reinit(self, mock_post): mock_post.return_value.status = 503 result = self.runner.invoke(ctl, ['restart', 'alpha'], input='now\ny\n') assert 'Failed: restart for' in result.output @@ -417,12 +404,10 @@ class TestCtl(unittest.TestCase): assert 'Failed: another restart is already' in result.output assert result.exit_code == 0 - @patch('patroni.ctl.get_dcs') - def test_remove(self, mock_get_dcs): - mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader + def test_remove(self): result = self.runner.invoke(ctl, ['remove', 'dummy'], input='\n') assert 'For Citus clusters the --group must me specified' in result.output - result = self.runner.invoke(ctl, ['-k', 'remove', 'alpha', '--group', '0'], input='alpha\nstandby') + result = self.runner.invoke(ctl, ['remove', 'alpha', '--group', '0'], input='alpha\nstandby') assert 'Please confirm' in result.output assert 'You are about to remove all' in result.output # Not typing an exact confirmation @@ -440,37 +425,36 @@ class TestCtl(unittest.TestCase): assert result.exit_code == 0 def test_ctl(self): - self.runner.invoke(ctl, ['list']) - result = self.runner.invoke(ctl, ['--help']) assert 'Usage:' in result.output def test_get_any_member(self): - for role in self.TEST_ROLES: - self.assertIsNone(get_any_member({}, get_cluster_initialized_without_leader(), None, role=role)) + with click.Context(click.Command('list')) as ctx: + ctx.obj = {'__config': {}} + for role in self.TEST_ROLES: + self.assertIsNone(get_any_member(get_cluster_initialized_without_leader(), None, role=role)) - m = get_any_member({}, get_cluster_initialized_with_leader(), None, role=role) - self.assertEqual(m.name, 'leader') + m = get_any_member(get_cluster_initialized_with_leader(), None, role=role) + self.assertEqual(m.name, 'leader') def test_get_all_members(self): - for role in self.TEST_ROLES: - self.assertEqual(list(get_all_members({}, get_cluster_initialized_without_leader(), None, role=role)), []) + with click.Context(click.Command('list')) as ctx: + ctx.obj = {'__config': {}} + for role in self.TEST_ROLES: + self.assertEqual(list(get_all_members(get_cluster_initialized_without_leader(), None, role=role)), []) - r = list(get_all_members({}, get_cluster_initialized_with_leader(), None, role=role)) + r = list(get_all_members(get_cluster_initialized_with_leader(), None, role=role)) + self.assertEqual(len(r), 1) + self.assertEqual(r[0].name, 'leader') + + r = list(get_all_members(get_cluster_initialized_with_leader(), None, role='replica')) self.assertEqual(len(r), 1) - self.assertEqual(r[0].name, 'leader') + self.assertEqual(r[0].name, 'other') - r = list(get_all_members({}, get_cluster_initialized_with_leader(), None, role='replica')) - self.assertEqual(len(r), 1) - self.assertEqual(r[0].name, 'other') - - self.assertEqual(len(list(get_all_members({}, get_cluster_initialized_without_leader(), - None, role='replica'))), 2) - - @patch('patroni.ctl.get_dcs') - def test_members(self, mock_get_dcs): - mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader + self.assertEqual(len(list(get_all_members(get_cluster_initialized_without_leader(), + None, role='replica'))), 2) + def test_members(self): result = self.runner.invoke(ctl, ['list']) assert '127.0.0.1' in result.output assert result.exit_code == 0 @@ -479,121 +463,94 @@ class TestCtl(unittest.TestCase): result = self.runner.invoke(ctl, ['list', '--group', '0']) assert 'Citus cluster: alpha (group: 0, 12345678901) -' in result.output - with patch('patroni.ctl.load_config', Mock(return_value={'scope': 'alpha'})): + config = get_default_config() + del config['citus'] + with patch('patroni.ctl.load_config', Mock(return_value=config)): result = self.runner.invoke(ctl, ['list']) assert 'Cluster: alpha (12345678901) -' in result.output with patch('patroni.ctl.load_config', Mock(return_value={})): self.runner.invoke(ctl, ['list']) - @patch('patroni.ctl.get_dcs') - def test_list_extended(self, mock_get_dcs): - mock_get_dcs.return_value = self.e - cluster = get_cluster_initialized_with_leader(sync=('leader', 'other')) - mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster) - + def test_list_extended(self): result = self.runner.invoke(ctl, ['list', 'dummy', '--extended', '--timestamp']) assert '2100' in result.output assert 'Scheduled restart' in result.output - @patch('patroni.ctl.get_dcs') - def test_topology(self, mock_get_dcs): - mock_get_dcs.return_value = self.e + def test_topology(self): cluster = get_cluster_initialized_with_leader() - cascade_member = Member(0, 'cascade', 28, {'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5437/postgres', - 'api_url': 'http://127.0.0.1:8012/patroni', - 'state': 'running', - 'tags': {'replicatefrom': 'other'}, - }) - cascade_member_wrong_tags = Member(0, 'wrong_cascade', 28, - {'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5438/postgres', - 'api_url': 'http://127.0.0.1:8013/patroni', - 'state': 'running', - 'tags': {'replicatefrom': 'nonexistinghost'}, - }) - cluster.members.append(cascade_member) - cluster.members.append(cascade_member_wrong_tags) - mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster) - result = self.runner.invoke(ctl, ['topology', 'dummy']) - assert '+\n| 0 | leader | 127.0.0.1:5435 | Leader |' in result.output - assert '|\n| 0 | + other | 127.0.0.1:5436 | Replica |' in result.output - assert '|\n| 0 | + cascade | 127.0.0.1:5437 | Replica |' in result.output - assert '|\n| 0 | + wrong_cascade | 127.0.0.1:5438 | Replica |' in result.output + cluster.members.append(Member(0, 'cascade', 28, + {'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5437/postgres', + 'api_url': 'http://127.0.0.1:8012/patroni', 'state': 'running', + 'tags': {'replicatefrom': 'other'}})) + cluster.members.append(Member(0, 'wrong_cascade', 28, + {'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5438/postgres', + 'api_url': 'http://127.0.0.1:8013/patroni', 'state': 'running', + 'tags': {'replicatefrom': 'nonexistinghost'}})) + with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=cluster)): + result = self.runner.invoke(ctl, ['topology', 'dummy']) + assert '+\n| 0 | leader | 127.0.0.1:5435 | Leader |' in result.output + assert '|\n| 0 | + other | 127.0.0.1:5436 | Replica |' in result.output + assert '|\n| 0 | + cascade | 127.0.0.1:5437 | Replica |' in result.output + assert '|\n| 0 | + wrong_cascade | 127.0.0.1:5438 | Replica |' in result.output - cluster = get_cluster_initialized_without_leader() - mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster) - result = self.runner.invoke(ctl, ['topology', 'dummy']) - assert '+\n| 0 | + leader | 127.0.0.1:5435 | Replica |' in result.output - assert '|\n| 0 | + other | 127.0.0.1:5436 | Replica |' in result.output - - @patch('patroni.ctl.get_dcs') - @patch.object(PoolManager, 'request', Mock(return_value=MockResponse())) - def test_flush_restart(self, mock_get_dcs): - mock_get_dcs.return_value = self.e - mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader + with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_without_leader())): + result = self.runner.invoke(ctl, ['topology', 'dummy']) + assert '+\n| 0 | + leader | 127.0.0.1:5435 | Replica |' in result.output + assert '|\n| 0 | + other | 127.0.0.1:5436 | Replica |' in result.output + @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) + def test_flush_restart(self): for role in self.TEST_ROLES: - result = self.runner.invoke(ctl, ['-k', 'flush', 'dummy', 'restart', '-r', role], input='y') + result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '-r', role], input='y') assert 'No scheduled restart' in result.output result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force']) assert 'Success: flush scheduled restart' in result.output - with patch.object(PoolManager, 'request', return_value=MockResponse(404)): + with patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse(404))): result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force']) assert 'Failed: flush scheduled restart' in result.output - @patch('patroni.ctl.get_dcs') - @patch.object(PoolManager, 'request', Mock(return_value=MockResponse())) - def test_flush_switchover(self, mock_get_dcs): - mock_get_dcs.return_value = self.e - - mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader - result = self.runner.invoke(ctl, ['flush', 'dummy', 'switchover']) - assert 'No pending scheduled switchover' in result.output + def test_flush_switchover(self): + with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())): + result = self.runner.invoke(ctl, ['flush', 'dummy', 'switchover']) + assert 'No pending scheduled switchover' in result.output scheduled_at = datetime.now(tzutc) + timedelta(seconds=600) - mock_get_dcs.return_value.get_cluster = Mock( - return_value=get_cluster_initialized_with_leader(Failover(1, 'a', 'b', scheduled_at))) - result = self.runner.invoke(ctl, ['flush', 'dummy', 'switchover']) - assert result.output.startswith('Success: ') + with patch('patroni.dcs.AbstractDCS.get_cluster', + Mock(return_value=get_cluster_initialized_with_leader(Failover(1, 'a', 'b', scheduled_at)))): + result = self.runner.invoke(ctl, ['-k', 'flush', 'dummy', 'switchover']) + assert result.output.startswith('Success: ') - mock_get_dcs.return_value.manual_failover = Mock() - with patch.object(PoolManager, 'request', side_effect=[MockResponse(409), Exception]): - result = self.runner.invoke(ctl, ['flush', 'dummy', 'switchover']) - assert 'Could not find any accessible member of cluster' in result.output + with patch('patroni.ctl.request_patroni', side_effect=[MockResponse(409), Exception]), \ + patch('patroni.dcs.AbstractDCS.manual_failover', Mock()): + result = self.runner.invoke(ctl, ['flush', 'dummy', 'switchover']) + assert 'Could not find any accessible member of cluster' in result.output - @patch.object(PoolManager, 'request') - @patch('patroni.ctl.get_dcs') @patch('patroni.ctl.polling_loop', Mock(return_value=[1])) - def test_pause_cluster(self, mock_get_dcs, mock_post): - mock_get_dcs.return_value = self.e - mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader + def test_pause_cluster(self): + with patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse(500))): + result = self.runner.invoke(ctl, ['pause', 'dummy']) + assert 'Failed' in result.output - mock_post.return_value.status = 500 - result = self.runner.invoke(ctl, ['pause', 'dummy']) - assert 'Failed' in result.output - - mock_post.return_value.status = 200 with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)): result = self.runner.invoke(ctl, ['pause', 'dummy']) assert 'Cluster is already paused' in result.output result = self.runner.invoke(ctl, ['pause', 'dummy', '--wait']) assert "'pause' request sent" in result.output - mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(), - get_cluster(None, None, [], None, None)]) - self.runner.invoke(ctl, ['pause', 'dummy', '--wait']) - member = Member(1, 'other', 28, {}) - mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(), - get_cluster(None, None, [member], None, None)]) - self.runner.invoke(ctl, ['pause', 'dummy', '--wait']) - @patch.object(PoolManager, 'request') - @patch('patroni.ctl.get_dcs') - def test_resume_cluster(self, mock_get_dcs, mock_post): - mock_get_dcs.return_value = self.e - mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader + with patch('patroni.dcs.AbstractDCS.get_cluster', + Mock(side_effect=[get_cluster_initialized_with_leader(), get_cluster(None, None, [], None, None)])): + self.runner.invoke(ctl, ['pause', 'dummy', '--wait']) + with patch('patroni.dcs.AbstractDCS.get_cluster', + Mock(side_effect=[get_cluster_initialized_with_leader(), + get_cluster(None, None, [Member(1, 'other', 28, {})], None, None)])): + self.runner.invoke(ctl, ['pause', 'dummy', '--wait']) + @patch('patroni.ctl.request_patroni') + @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) + def test_resume_cluster(self, mock_post): mock_post.return_value.status = 200 with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=False)): result = self.runner.invoke(ctl, ['resume', 'dummy']) @@ -701,67 +658,53 @@ class TestCtl(unittest.TestCase): with patch('shutil.which', Mock(return_value=e)): self.assertRaises(PatroniCtlException, invoke_editor, 'foo: bar\n', 'test') - @patch('patroni.ctl.get_dcs') - def test_show_config(self, mock_get_dcs): - mock_get_dcs.return_value = self.e - mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader + def test_show_config(self): self.runner.invoke(ctl, ['show-config', 'dummy']) - @patch('patroni.ctl.get_dcs') @patch('subprocess.call', Mock(return_value=0)) - def test_edit_config(self, mock_get_dcs): - mock_get_dcs.return_value = self.e - mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader - mock_get_dcs.return_value.set_config_value = Mock(return_value=False) + def test_edit_config(self): os.environ['EDITOR'] = 'true' self.runner.invoke(ctl, ['edit-config', 'dummy']) self.runner.invoke(ctl, ['edit-config', 'dummy', '-s', 'foo=bar']) self.runner.invoke(ctl, ['edit-config', 'dummy', '--replace', 'postgres0.yml']) self.runner.invoke(ctl, ['edit-config', 'dummy', '--apply', '-'], input='foo: bar') self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar') - mock_get_dcs.return_value.set_config_value.return_value = True - self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar') - mock_get_dcs.return_value.get_cluster = Mock(return_value=Cluster.empty()) - result = self.runner.invoke(ctl, ['edit-config', 'dummy']) - assert result.exit_code == 1 - assert 'The config key does not exist in the cluster dummy' in result.output + with patch('patroni.dcs.etcd.Etcd.set_config_value', Mock(return_value=True)): + self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar') + with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=Cluster.empty())): + result = self.runner.invoke(ctl, ['edit-config', 'dummy']) + assert result.exit_code == 1 + assert 'The config key does not exist in the cluster dummy' in result.output - @patch('patroni.ctl.get_dcs') - def test_version(self, mock_get_dcs): - mock_get_dcs.return_value = self.e - mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader - with patch.object(PoolManager, 'request') as mocked: - result = self.runner.invoke(ctl, ['version']) - assert 'patronictl version' in result.output - mocked.return_value.data = b'{"patroni":{"version":"1.2.3"},"server_version": 100001}' - result = self.runner.invoke(ctl, ['version', 'dummy']) - assert '1.2.3' in result.output - with patch.object(PoolManager, 'request', Mock(side_effect=Exception)): - result = self.runner.invoke(ctl, ['version', 'dummy']) - assert 'failed to get version' in result.output + @patch('patroni.ctl.request_patroni') + def test_version(self, mock_request): + result = self.runner.invoke(ctl, ['version']) + assert 'patronictl version' in result.output + mock_request.return_value.data = b'{"patroni":{"version":"1.2.3"},"server_version": 100001}' + result = self.runner.invoke(ctl, ['version', 'dummy']) + assert '1.2.3' in result.output + mock_request.side_effect = Exception + result = self.runner.invoke(ctl, ['version', 'dummy']) + assert 'failed to get version' in result.output - @patch('patroni.ctl.get_dcs') - def test_history(self, mock_get_dcs): - mock_get_dcs.return_value.get_cluster = Mock() - mock_get_dcs.return_value.get_cluster.return_value.history.lines = [[1, 67176, 'no recovery target specified']] - result = self.runner.invoke(ctl, ['history']) - assert 'Reason' in result.output + def test_history(self): + with patch('patroni.dcs.AbstractDCS.get_cluster') as mock_get_cluster: + mock_get_cluster.return_value.history.lines = [[1, 67176, 'no recovery target specified']] + result = self.runner.invoke(ctl, ['history']) + assert 'Reason' in result.output def test_format_pg_version(self): self.assertEqual(format_pg_version(100001), '10.1') self.assertEqual(format_pg_version(90605), '9.6.5') - @patch('patroni.ctl.get_dcs') - def test_get_members(self, mock_get_dcs): - mock_get_dcs.return_value = self.e - mock_get_dcs.return_value.get_cluster = get_cluster_not_initialized_without_leader - result = self.runner.invoke(ctl, ['reinit', 'dummy']) - assert "cluster doesn\'t have any members" in result.output + def test_get_members(self): + with patch('patroni.dcs.AbstractDCS.get_cluster', + Mock(return_value=get_cluster_not_initialized_without_leader())): + result = self.runner.invoke(ctl, ['reinit', 'dummy']) + assert "cluster doesn\'t have any members" in result.output @patch('time.sleep', Mock()) - @patch('patroni.ctl.get_dcs') - def test_reinit_wait(self, mock_get_dcs): - mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader + def test_reinit_wait(self): with patch.object(PoolManager, 'request') as mocked: mocked.side_effect = [Mock(data=s, status=200) for s in [b"reinitialize", b'{"state":"creating replica"}', b'{"state":"running"}']] From 5dab73553457d486474dd8ab53cf27fabdad6d7a Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 15 Nov 2023 11:25:46 +0100 Subject: [PATCH 15/19] Compatibility with antient mock (#2951) Just in case is someone still uses ubuntu 18.04 --- tests/test_barman_recover.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/tests/test_barman_recover.py b/tests/test_barman_recover.py index 4334d6ae..c0efda83 100644 --- a/tests/test_barman_recover.py +++ b/tests/test_barman_recover.py @@ -1,5 +1,6 @@ import logging -from mock import MagicMock, Mock, call, patch +import mock +from mock import MagicMock, Mock, patch import unittest from urllib3.exceptions import MaxRetryError @@ -156,18 +157,19 @@ class TestBarmanRecover(unittest.TestCase): "Maximum number of retries exceeded for method BarmanRecover._create_recovery_operation.") self.assertEqual(mock_sleep.call_count, self.br.max_retries) - mock_sleep.assert_has_calls([call(self.br.retry_wait)] * self.br.max_retries) + + mock_sleep.assert_has_calls([mock.call(self.br.retry_wait)] * self.br.max_retries) self.assertEqual(mock_logging.call_count, self.br.max_retries) for i in range(mock_logging.call_count): - call_args = mock_logging.mock_calls[i].args + call_args = mock_logging.call_args_list[i][0] self.assertEqual(len(call_args), 5) self.assertEqual(call_args[0], "Attempt %d of %d on method %s failed with %r.") self.assertEqual(call_args[1], i + 1) self.assertEqual(call_args[2], self.br.max_retries) self.assertEqual(call_args[3], "BarmanRecover._create_recovery_operation") self.assertIsInstance(call_args[4], KeyError) - self.assertEqual(repr(call_args[4]), "KeyError('operation_id')") + self.assertEqual(call_args[4].args, ('operation_id',)) @patch("logging.warning") @patch("time.sleep") @@ -190,18 +192,18 @@ class TestBarmanRecover(unittest.TestCase): "Maximum number of retries exceeded for method BarmanRecover._get_recovery_operation_status.") self.assertEqual(mock_sleep.call_count, self.br.max_retries) - mock_sleep.assert_has_calls([call(self.br.retry_wait)] * self.br.max_retries) + mock_sleep.assert_has_calls([mock.call(self.br.retry_wait)] * self.br.max_retries) self.assertEqual(mock_logging.call_count, self.br.max_retries) for i in range(mock_logging.call_count): - call_args = mock_logging.mock_calls[i].args + call_args = mock_logging.call_args_list[i][0] self.assertEqual(len(call_args), 5) self.assertEqual(call_args[0], "Attempt %d of %d on method %s failed with %r.") self.assertEqual(call_args[1], i + 1) self.assertEqual(call_args[2], self.br.max_retries) self.assertEqual(call_args[3], "BarmanRecover._get_recovery_operation_status") self.assertIsInstance(call_args[4], KeyError) - self.assertEqual(repr(call_args[4]), "KeyError('status')") + self.assertEqual(call_args[4].args, ('status',)) @patch.object(BarmanRecover, "_get_recovery_operation_status") @patch("time.sleep") @@ -232,16 +234,16 @@ class TestBarmanRecover(unittest.TestCase): mock_create_op.assert_called_once() self.assertEqual(mock_get_status.call_count, 21) - mock_get_status.assert_has_calls([call("some_id")] * 21) + mock_get_status.assert_has_calls([mock.call("some_id")] * 21) self.assertEqual(mock_log_info.call_count, 21) - mock_log_info.assert_has_calls([call("Created the recovery operation with ID %s", "some_id")] - + [call("Recovery operation %s is still in progress", "some_id")] * 20) + mock_log_info.assert_has_calls([mock.call("Created the recovery operation with ID %s", "some_id")] + + [mock.call("Recovery operation %s is still in progress", "some_id")] * 20) mock_log_critical.assert_not_called() self.assertEqual(mock_sleep.call_count, 20) - mock_sleep.assert_has_calls([call(LOOP_WAIT)] * 20) + mock_sleep.assert_has_calls([mock.call(LOOP_WAIT)] * 20) # failed fast restore mock_create_op.reset_mock() @@ -271,16 +273,16 @@ class TestBarmanRecover(unittest.TestCase): mock_create_op.assert_called_once() self.assertEqual(mock_get_status.call_count, 21) - mock_get_status.assert_has_calls([call("some_id")] * 21) + mock_get_status.assert_has_calls([mock.call("some_id")] * 21) self.assertEqual(mock_log_info.call_count, 21) - mock_log_info.assert_has_calls([call("Created the recovery operation with ID %s", "some_id")] - + [call("Recovery operation %s is still in progress", "some_id")] * 20) + mock_log_info.assert_has_calls([mock.call("Created the recovery operation with ID %s", "some_id")] + + [mock.call("Recovery operation %s is still in progress", "some_id")] * 20) mock_log_critical.assert_not_called() self.assertEqual(mock_sleep.call_count, 20) - mock_sleep.assert_has_calls([call(LOOP_WAIT)] * 20) + mock_sleep.assert_has_calls([mock.call(LOOP_WAIT)] * 20) # create retries exceeded mock_log_info.reset_mock() From 70b0991e6a7153a67375ca52c67696739a107506 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 20 Nov 2023 10:22:52 +0100 Subject: [PATCH 16/19] Bump pyright to 1.1.336 (#2952) and fix newly reported issues --- .github/workflows/tests.yaml | 2 +- patroni/validator.py | 30 +++++++++++++++++++----------- typings/urllib3/_collections.pyi | 2 +- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 81119f25..df911e54 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -174,7 +174,7 @@ jobs: - uses: jakebailey/pyright-action@v1 with: - version: 1.1.333 + version: 1.1.336 docs: runs-on: ubuntu-latest diff --git a/patroni/validator.py b/patroni/validator.py index 016f4b2e..d0b168be 100644 --- a/patroni/validator.py +++ b/patroni/validator.py @@ -9,7 +9,7 @@ import os import shutil import socket -from typing import Any, Dict, Union, Iterator, List, Optional as OptionalType, Tuple +from typing import Any, Dict, Union, Iterator, List, Optional as OptionalType, Tuple, TYPE_CHECKING from .collections import CaseInsensitiveSet @@ -200,6 +200,8 @@ def get_bin_name(bin_name: str) -> str: :returns: value of ``postgresql.bin_name[*bin_name*]``, if present, otherwise *bin_name*. """ + if TYPE_CHECKING: # pragma: no cover + assert isinstance(schema.data, dict) return (schema.data.get('postgresql', {}).get('bin_name', {}) or {}).get(bin_name, bin_name) @@ -239,6 +241,8 @@ def validate_data_dir(data_dir: str) -> bool: if not os.path.isdir(os.path.join(data_dir, waldir)): raise ConfigParseError("data dir for the cluster is not empty, but doesn't contain" " \"{}\" directory".format(waldir)) + if TYPE_CHECKING: # pragma: no cover + assert isinstance(schema.data, dict) bin_dir = schema.data.get("postgresql", {}).get("bin_dir", None) major_version = get_major_version(bin_dir, get_bin_name('postgres')) if pgversion != major_version: @@ -274,6 +278,8 @@ def validate_binary_name(bin_name: str) -> bool: """ if not bin_name: raise ConfigParseError("is an empty string") + if TYPE_CHECKING: # pragma: no cover + assert isinstance(schema.data, dict) bin_dir = schema.data.get('postgresql', {}).get('bin_dir', None) if not shutil.which(bin_name, path=bin_dir): raise ConfigParseError(f"does not contain '{bin_name}' in '{bin_dir or '$PATH'}'") @@ -523,7 +529,7 @@ class Schema(object): * :class:`dict`: dictionary representing the YAML configuration tree. """ - def __init__(self, validator: Any) -> None: + def __init__(self, validator: Union[Dict[Any, Any], List[Any], Any]) -> None: """Create a :class:`Schema` object. .. note:: @@ -614,7 +620,7 @@ class Schema(object): errors.append(str(i)) return errors - def validate(self, data: Any) -> Iterator[Result]: + def validate(self, data: Union[Dict[Any, Any], Any]) -> Iterator[Result]: """Perform all validations from the schema against the given configuration. It first checks that *data* argument type is compliant with the type of ``validator`` attribute. @@ -638,11 +644,8 @@ class Schema(object): # iterable objects in the structure, until we eventually reach a leaf node to validate its value. if isinstance(self.validator, str): yield Result(isinstance(self.data, str), "is not a string", level=1, data=self.data) - elif issubclass(type(self.validator), type): - validator = self.validator - if self.validator == str: - validator = str - yield Result(isinstance(self.data, validator), + elif isinstance(self.validator, type): + yield Result(isinstance(self.data, self.validator), "is not {}".format(_get_type_name(self.validator)), level=1, data=self.data) elif callable(self.validator): if hasattr(self.validator, "expected_type"): @@ -689,7 +692,7 @@ class Schema(object): for v in Schema(self.validator[0]).validate(value): yield Result(v.status, v.error, path=(str(key) + ("." + v.path if v.path else "")), level=v.level, data=value) - elif isinstance(self.validator, Directory): + elif isinstance(self.validator, Directory) and isinstance(self.data, str): yield from self.validator.validate(self.data) elif isinstance(self.validator, Or): yield from self.iter_or() @@ -701,6 +704,9 @@ class Schema(object): """ # One key in `validator` attribute (`key` variable) can be mapped to one or more keys in `data` attribute (`d` # variable), depending on the `key` type. + if TYPE_CHECKING: # pragma: no cover + assert isinstance(self.validator, dict) + assert isinstance(self.data, dict) for key in self.validator.keys(): if isinstance(key, AtMostOne) and len(list(self._data_key(key))) > 1: yield Result(False, f"Multiple of {key.args} provided") @@ -730,6 +736,8 @@ class Schema(object): :yields: objects with the error message related to the failure, if any check fails. """ + if TYPE_CHECKING: # pragma: no cover + assert isinstance(self.validator, Or) results: List[Result] = [] for a in self.validator.args: r: List[Result] = [] @@ -766,7 +774,7 @@ class Schema(object): yield key.name # If the key was defined as an `Or` object in `validator` attribute, then each of its values are the keys to # access the `data` dictionary. - elif isinstance(key, Or): + elif isinstance(key, Or) and isinstance(self.data, dict): # At least one of the `Or` entries should be available in the `data` dictionary. If we find at least one of # them in `data`, then we return all found entries so the caller method can validate them all. if any([item in self.data for item in key.args]): @@ -780,7 +788,7 @@ class Schema(object): yield item # If the key was defined as a `AtMostOne` object in `validator` attribute, then each of its values # are the keys to access the `data` dictionary. - elif isinstance(key, AtMostOne): + elif isinstance(key, AtMostOne) and isinstance(self.data, dict): # Yield back all of the entries from the `data` dictionary, each will be validated and then counted # to inform us if we've provided too many for item in key.args: diff --git a/typings/urllib3/_collections.pyi b/typings/urllib3/_collections.pyi index b7a8cebe..52b9f577 100644 --- a/typings/urllib3/_collections.pyi +++ b/typings/urllib3/_collections.pyi @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, MutableMapping class HTTPHeaderDict(MutableMapping[str, str]): def __init__(self, headers=None, **kwargs) -> None: ... def __setitem__(self, key, val) -> None: ... From ac6f6ae1c2075b5f9e47b2be5bf9c2c4a9f46afd Mon Sep 17 00:00:00 2001 From: Ali Mehraji Date: Wed, 22 Nov 2023 11:25:51 +0330 Subject: [PATCH 17/19] Add ETCDCTL_API=3 env to Dockerfiles and update docker/README.md (#2946) --- Dockerfile | 1 + Dockerfile.citus | 1 + docker-compose-citus.yml | 2 - docker/README.md | 334 +++++++++++++++++++-------------------- 4 files changed, 169 insertions(+), 169 deletions(-) diff --git a/Dockerfile b/Dockerfile index c5b927ee..3e638518 100644 --- a/Dockerfile +++ b/Dockerfile @@ -143,6 +143,7 @@ ARG PGBIN=/usr/lib/postgresql/$PG_MAJOR/bin ENV LC_ALL=$LC_ALL LANG=$LANG EDITOR=/usr/bin/editor ENV PGDATA=$PGDATA PATH=$PATH:$PGBIN +ENV ETCDCTL_API=3 COPY patroni /patroni/ COPY extras/confd/conf.d/haproxy.toml /etc/confd/conf.d/ diff --git a/Dockerfile.citus b/Dockerfile.citus index 7e6ec18c..f52a36e7 100644 --- a/Dockerfile.citus +++ b/Dockerfile.citus @@ -164,6 +164,7 @@ ARG PGBIN=/usr/lib/postgresql/$PG_MAJOR/bin ENV LC_ALL=$LC_ALL LANG=$LANG EDITOR=/usr/bin/editor ENV PGDATA=$PGDATA PATH=$PATH:$PGBIN +ENV ETCDCTL_API=3 COPY patroni /patroni/ COPY extras/confd/conf.d/haproxy.toml /etc/confd/conf.d/ diff --git a/docker-compose-citus.yml b/docker-compose-citus.yml index 7ff2a2c5..cd63d583 100644 --- a/docker-compose-citus.yml +++ b/docker-compose-citus.yml @@ -19,7 +19,6 @@ services: image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] environment: - ETCDCTL_API: 3 ETCD_LISTEN_PEER_URLS: http://0.0.0.0:2380 ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379 ETCD_INITIAL_CLUSTER: etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380 @@ -53,7 +52,6 @@ services: - "5001:5001" # Load-balancing across workers primaries command: haproxy environment: &haproxy_env - ETCDCTL_API: 3 ETCDCTL_ENDPOINTS: http://etcd1:2379,http://etcd2:2379,http://etcd3:2379 PATRONI_ETCD3_HOSTS: "'etcd1:2379','etcd2:2379','etcd3:2379'" PATRONI_SCOPE: demo diff --git a/docker/README.md b/docker/README.md index da87842f..0b7f3d58 100644 --- a/docker/README.md +++ b/docker/README.md @@ -19,102 +19,97 @@ The haproxy listens on ports 5000 (connects to the primary) and 5001 (does load- Example session: - $ docker-compose up -d - Creating demo-haproxy ... - Creating demo-patroni2 ... - Creating demo-patroni1 ... - Creating demo-patroni3 ... - Creating demo-etcd2 ... - Creating demo-etcd1 ... - Creating demo-etcd3 ... - Creating demo-haproxy - Creating demo-patroni2 - Creating demo-patroni1 - Creating demo-patroni3 - Creating demo-etcd1 - Creating demo-etcd2 - Creating demo-etcd2 ... done + $ docker compose up -d + ✔ Network patroni_demo Created + ✔ Container demo-etcd1 Started + ✔ Container demo-haproxy Started + ✔ Container demo-patroni1 Started + ✔ Container demo-patroni2 Started + ✔ Container demo-patroni3 Started + ✔ Container demo-etcd2 Started + ✔ Container demo-etcd3 Started $ docker ps - CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - 5b7a90b4cfbf patroni "/bin/sh /entrypoint…" 29 seconds ago Up 27 seconds demo-etcd2 - e30eea5222f2 patroni "/bin/sh /entrypoint…" 29 seconds ago Up 27 seconds demo-etcd1 - 83bcf3cb208f patroni "/bin/sh /entrypoint…" 29 seconds ago Up 27 seconds demo-etcd3 - 922532c56e7d patroni "/bin/sh /entrypoint…" 29 seconds ago Up 28 seconds demo-patroni3 - 14f875e445f3 patroni "/bin/sh /entrypoint…" 29 seconds ago Up 28 seconds demo-patroni2 - 110d1073b383 patroni "/bin/sh /entrypoint…" 29 seconds ago Up 28 seconds demo-patroni1 - 5af5e6e36028 patroni "/bin/sh /entrypoint…" 29 seconds ago Up 28 seconds 0.0.0.0:5000-5001->5000-5001/tcp demo-haproxy + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + a37bcec56726 patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes demo-etcd3 + 034ab73868a8 patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes demo-patroni2 + 03837736f710 patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes demo-patroni3 + 22815c3d85b3 patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes demo-etcd2 + 814b4304d132 patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes 0.0.0.0:5000-5001->5000-5001/tcp, :::5000-5001->5000-5001/tcp demo-haproxy + 6375b0ba2d0a patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes demo-patroni1 + aef8bf3ee91f patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes demo-etcd1 $ docker logs demo-patroni1 - 2019-02-20 08:19:32,714 INFO: Failed to import patroni.dcs.consul - 2019-02-20 08:19:32,737 INFO: Selected new etcd server http://etcd3:2379 - 2019-02-20 08:19:35,140 INFO: Lock owner: None; I am patroni1 - 2019-02-20 08:19:35,174 INFO: trying to bootstrap a new cluster + 2023-11-21 09:04:33,547 INFO: Selected new etcd server http://172.29.0.3:2379 + 2023-11-21 09:04:33,605 INFO: Lock owner: None; I am patroni1 + 2023-11-21 09:04:33,693 INFO: trying to bootstrap a new cluster ... - 2019-02-20 08:19:39,310 INFO: postmaster pid=37 - 2019-02-20 08:19:39.314 UTC [37] LOG: listening on IPv4 address "0.0.0.0", port 5432 - 2019-02-20 08:19:39.321 UTC [37] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432" - 2019-02-20 08:19:39.353 UTC [39] LOG: database system was shut down at 2019-02-20 08:19:36 UTC - 2019-02-20 08:19:39.354 UTC [40] FATAL: the database system is starting up - localhost:5432 - rejecting connections - 2019-02-20 08:19:39.369 UTC [37] LOG: database system is ready to accept connections + 2023-11-21 09:04:34.920 UTC [43] LOG: starting PostgreSQL 15.5 (Debian 15.5-1.pgdg120+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14) 12.2.0, 64-bit + 2023-11-21 09:04:34.921 UTC [43] LOG: listening on IPv4 address "0.0.0.0", port 5432 + 2023-11-21 09:04:34,922 INFO: postmaster pid=43 + 2023-11-21 09:04:34.922 UTC [43] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432" + 2023-11-21 09:04:34.925 UTC [47] LOG: database system was shut down at 2023-11-21 09:04:34 UTC + 2023-11-21 09:04:34.928 UTC [43] LOG: database system is ready to accept connections localhost:5432 - accepting connections - 2019-02-20 08:19:39,383 INFO: establishing a new patroni connection to the postgres cluster - 2019-02-20 08:19:39,408 INFO: running post_bootstrap - 2019-02-20 08:19:39,432 WARNING: Could not activate Linux watchdog device: "Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog'" - 2019-02-20 08:19:39,515 INFO: initialized a new cluster - 2019-02-20 08:19:49,424 INFO: Lock owner: patroni1; I am patroni1 - 2019-02-20 08:19:49,447 INFO: Lock owner: patroni1; I am patroni1 - 2019-02-20 08:19:49,480 INFO: no action. i am the leader with the lock - 2019-02-20 08:19:59,422 INFO: Lock owner: patroni1; I am patroni1 + localhost:5432 - accepting connections + 2023-11-21 09:04:34,938 INFO: establishing a new patroni heartbeat connection to postgres + 2023-11-21 09:04:34,992 INFO: running post_bootstrap + 2023-11-21 09:04:35,004 WARNING: User creation via "bootstrap.users" will be removed in v4.0.0 + 2023-11-21 09:04:35,009 WARNING: Could not activate Linux watchdog device: Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog' + 2023-11-21 09:04:35,189 INFO: initialized a new cluster + 2023-11-21 09:04:35,328 INFO: no action. I am (patroni1), the leader with the lock + 2023-11-21 09:04:43,824 INFO: establishing a new patroni restapi connection to postgres + 2023-11-21 09:04:45,322 INFO: no action. I am (patroni1), the leader with the lock + 2023-11-21 09:04:55,320 INFO: no action. I am (patroni1), the leader with the lock + ... $ docker exec -ti demo-patroni1 bash postgres@patroni1:~$ patronictl list - +---------+----------+------------+--------+---------+----+-----------+ - | Cluster | Member | Host | Role | State | TL | Lag in MB | - +---------+----------+------------+--------+---------+----+-----------+ - | demo | patroni1 | 172.22.0.3 | Leader | running | 1 | 0 | - | demo | patroni2 | 172.22.0.7 | | running | 1 | 0 | - | demo | patroni3 | 172.22.0.4 | | running | 1 | 0 | - +---------+----------+------------+--------+---------+----+-----------+ + + Cluster: demo (7303838734793224214) --------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +----------+------------+---------+-----------+----+-----------+ + | patroni1 | 172.29.0.2 | Leader | running | 1 | | + | patroni2 | 172.29.0.6 | Replica | streaming | 1 | 0 | + | patroni3 | 172.29.0.5 | Replica | streaming | 1 | 0 | + +----------+------------+---------+-----------+----+-----------+ postgres@patroni1:~$ etcdctl get --keys-only --prefix /service/demo /service/demo/config /service/demo/initialize /service/demo/leader - /service/demo/members/ /service/demo/members/patroni1 /service/demo/members/patroni2 /service/demo/members/patroni3 - /service/demo/optime/ - /service/demo/optime/leader + /service/demo/status postgres@patroni1:~$ etcdctl member list - 1bab629f01fa9065: name=etcd3 peerURLs=http://etcd3:2380 clientURLs=http://etcd3:2379 isLeader=false - 8ecb6af518d241cc: name=etcd2 peerURLs=http://etcd2:2380 clientURLs=http://etcd2:2379 isLeader=true - b2e169fcb8a34028: name=etcd1 peerURLs=http://etcd1:2380 clientURLs=http://etcd1:2379 isLeader=false + 2bf3e2ceda5d5960, started, etcd2, http://etcd2:2380, http://172.29.0.3:2379 + 55b3264e129c7005, started, etcd3, http://etcd3:2380, http://172.29.0.7:2379 + acce7233f8ec127e, started, etcd1, http://etcd1:2380, http://172.29.0.8:2379 + + postgres@patroni1:~$ exit $ docker exec -ti demo-haproxy bash postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -W Password: postgres - psql (11.2 (Ubuntu 11.2-1.pgdg18.04+1), server 10.7 (Debian 10.7-1.pgdg90+1)) + psql (15.5 (Debian 15.5-1.pgdg120+1)) Type "help" for help. - localhost/postgres=# select pg_is_in_recovery(); + postgres=# SELECT pg_is_in_recovery(); pg_is_in_recovery ─────────────────── f (1 row) - localhost/postgres=# \q + postgres=# \q - $postgres@haproxy:~ psql -h localhost -p 5001 -U postgres -W + postgres@haproxy:~$ psql -h localhost -p 5001 -U postgres -W Password: postgres - psql (11.2 (Ubuntu 11.2-1.pgdg18.04+1), server 10.7 (Debian 10.7-1.pgdg90+1)) + psql (15.5 (Debian 15.5-1.pgdg120+1)) Type "help" for help. - localhost/postgres=# select pg_is_in_recovery(); + postgres=# SELECT pg_is_in_recovery(); pg_is_in_recovery ─────────────────── t @@ -127,81 +122,86 @@ The haproxy listens on ports 5000 (connects to the coordinator primary) and 5001 Example session: - $ docker-compose -f docker-compose-citus.yml up -d - Creating demo-work2-1 ... done - Creating demo-work1-1 ... done - Creating demo-etcd2 ... done - Creating demo-etcd1 ... done - Creating demo-coord3 ... done - Creating demo-etcd3 ... done - Creating demo-coord1 ... done - Creating demo-haproxy ... done - Creating demo-work2-2 ... done - Creating demo-coord2 ... done - Creating demo-work1-2 ... done + $ docker compose -f docker-compose-citus.yml up -d + ✔ Network patroni_demo Created + ✔ Container demo-coord2 Started + ✔ Container demo-work2-2 Started + ✔ Container demo-etcd1 Started + ✔ Container demo-haproxy Started + ✔ Container demo-work1-1 Started + ✔ Container demo-work2-1 Started + ✔ Container demo-work1-2 Started + ✔ Container demo-coord1 Started + ✔ Container demo-etcd3 Started + ✔ Container demo-coord3 Started + ✔ Container demo-etcd2 Started + $ docker ps - CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - 852d8885a612 patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 3 seconds demo-coord3 - cdd692f947ab patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 3 seconds demo-work1-2 - 9f4e340b36da patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 3 seconds demo-etcd3 - d69c129a960a patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 4 seconds demo-etcd1 - c5849689b8cd patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 4 seconds demo-coord1 - c9d72bd6217d patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 3 seconds demo-work2-1 - 24b1b43efa05 patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 4 seconds demo-coord2 - cb0cc2b4ca0a patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 3 seconds demo-work2-2 - 9796c6b8aad5 patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 5 seconds demo-work1-1 - 8baccd74dcae patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 4 seconds demo-etcd2 - 353ec62a0187 patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 4 seconds 0.0.0.0:5000-5001->5000-5001/tcp demo-haproxy + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 79c95492fac9 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-etcd3 + 77eb82d0f0c1 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-work2-1 + 03dacd7267ef patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-etcd1 + db9206c66f85 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-etcd2 + 9a0fef7b7dd4 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-work1-2 + f06b031d99dc patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-work2-2 + f7c58545f314 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-coord2 + 383f9e7e188a patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-work1-1 + f02e96dcc9d6 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-coord3 + 6945834b7056 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-coord1 + b96ca42f785d patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes 0.0.0.0:5000-5001->5000-5001/tcp, :::5000-5001->5000-5001/tcp demo-haproxy + $ docker logs demo-coord1 - 2023-01-05 15:09:31,295 INFO: Selected new etcd server http://172.27.0.4:2379 - 2023-01-05 15:09:31,388 INFO: Lock owner: None; I am coord1 - 2023-01-05 15:09:31,501 INFO: trying to bootstrap a new cluster + 2023-11-21 09:36:14,293 INFO: Selected new etcd server http://172.30.0.4:2379 + 2023-11-21 09:36:14,390 INFO: Lock owner: None; I am coord1 + 2023-11-21 09:36:14,478 INFO: trying to bootstrap a new cluster ... - 2023-01-05 15:09:45,096 INFO: postmaster pid=39 + 2023-11-21 09:36:16,475 INFO: postmaster pid=52 localhost:5432 - no response - 2023-01-05 15:09:45.137 UTC [39] LOG: starting PostgreSQL 15.1 (Debian 15.1-1.pgdg110+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 10.2.1-6) 10.2.1 20210110, 64-bit - 2023-01-05 15:09:45.137 UTC [39] LOG: listening on IPv4 address "0.0.0.0", port 5432 - 2023-01-05 15:09:45.152 UTC [39] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432" - 2023-01-05 15:09:45.177 UTC [43] LOG: database system was shut down at 2023-01-05 15:09:32 UTC - 2023-01-05 15:09:45.193 UTC [39] LOG: database system is ready to accept connections + 2023-11-21 09:36:16.495 UTC [52] LOG: starting PostgreSQL 15.5 (Debian 15.5-1.pgdg120+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14) 12.2.0, 64-bit + 2023-11-21 09:36:16.495 UTC [52] LOG: listening on IPv4 address "0.0.0.0", port 5432 + 2023-11-21 09:36:16.496 UTC [52] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432" + 2023-11-21 09:36:16.498 UTC [56] LOG: database system was shut down at 2023-11-21 09:36:15 UTC + 2023-11-21 09:36:16.501 UTC [52] LOG: database system is ready to accept connections localhost:5432 - accepting connections localhost:5432 - accepting connections - 2023-01-05 15:09:46,139 INFO: establishing a new patroni connection to the postgres cluster - 2023-01-05 15:09:46,208 INFO: running post_bootstrap - 2023-01-05 15:09:47.209 UTC [55] LOG: starting maintenance daemon on database 16386 user 10 - 2023-01-05 15:09:47.209 UTC [55] CONTEXT: Citus maintenance daemon for database 16386 user 10 - 2023-01-05 15:09:47,215 WARNING: Could not activate Linux watchdog device: "Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog'" - 2023-01-05 15:09:47.446 UTC [41] LOG: checkpoint starting: immediate force wait - 2023-01-05 15:09:47,466 INFO: initialized a new cluster - 2023-01-05 15:09:47,594 DEBUG: query(SELECT nodeid, groupid, nodename, nodeport, noderole FROM pg_catalog.pg_dist_node WHERE noderole = 'primary', ()) - 2023-01-05 15:09:47,594 INFO: establishing a new patroni connection to the postgres cluster - 2023-01-05 15:09:47,467 INFO: Lock owner: coord1; I am coord1 - 2023-01-05 15:09:47,613 DEBUG: query(SELECT pg_catalog.citus_set_coordinator_host(%s, %s, 'primary', 'default'), ('172.27.0.6', 5432)) - 2023-01-05 15:09:47,924 INFO: no action. I am (coord1), the leader with the lock - 2023-01-05 15:09:51.282 UTC [41] LOG: checkpoint complete: wrote 1086 buffers (53.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.029 s, sync=3.746 s, total=3.837 s; sync files=280, longest=0.028 s, average=0.014 s; distance=8965 kB, estimate=8965 kB - 2023-01-05 15:09:51.283 UTC [41] LOG: checkpoint starting: immediate force wait - 2023-01-05 15:09:51.495 UTC [41] LOG: checkpoint complete: wrote 18 buffers (0.9%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.044 s, sync=0.091 s, total=0.212 s; sync files=15, longest=0.015 s, average=0.007 s; distance=67 kB, estimate=8076 kB - 2023-01-05 15:09:57,467 INFO: Lock owner: coord1; I am coord1 - 2023-01-05 15:09:57,569 INFO: Assigning synchronous standby status to ['coord3'] + 2023-11-21 09:36:17,509 INFO: establishing a new patroni heartbeat connection to postgres + 2023-11-21 09:36:17,569 INFO: running post_bootstrap + 2023-11-21 09:36:17,593 WARNING: User creation via "bootstrap.users" will be removed in v4.0.0 + 2023-11-21 09:36:17,783 INFO: establishing a new patroni restapi connection to postgres + 2023-11-21 09:36:17,969 WARNING: Could not activate Linux watchdog device: Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog' + 2023-11-21 09:36:17.969 UTC [70] LOG: starting maintenance daemon on database 16386 user 10 + 2023-11-21 09:36:17.969 UTC [70] CONTEXT: Citus maintenance daemon for database 16386 user 10 + 2023-11-21 09:36:18.159 UTC [54] LOG: checkpoint starting: immediate force wait + 2023-11-21 09:36:18,162 INFO: initialized a new cluster + 2023-11-21 09:36:18,164 INFO: Lock owner: coord1; I am coord1 + 2023-11-21 09:36:18,297 INFO: Enabled synchronous replication + 2023-11-21 09:36:18,298 DEBUG: Adding the new task: PgDistNode(nodeid=None,group=0,host=172.30.0.3,port=5432,event=after_promote) + 2023-11-21 09:36:18,298 DEBUG: Adding the new task: PgDistNode(nodeid=None,group=1,host=172.30.0.7,port=5432,event=after_promote) + 2023-11-21 09:36:18,298 DEBUG: Adding the new task: PgDistNode(nodeid=None,group=2,host=172.30.0.8,port=5432,event=after_promote) + 2023-11-21 09:36:18,299 DEBUG: query(SELECT nodeid, groupid, nodename, nodeport, noderole FROM pg_catalog.pg_dist_node WHERE noderole = 'primary', ()) + 2023-11-21 09:36:18,299 INFO: establishing a new patroni citus connection to postgres + 2023-11-21 09:36:18,323 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.30.0.7', 5432, 1)) + 2023-11-21 09:36:18,361 INFO: no action. I am (coord1), the leader with the lock + 2023-11-21 09:36:18,393 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.30.0.8', 5432, 2)) + 2023-11-21 09:36:28,164 INFO: Lock owner: coord1; I am coord1 + 2023-11-21 09:36:28,251 INFO: Assigning synchronous standby status to ['coord3'] server signaled - 2023-01-05 15:09:57.574 UTC [39] LOG: received SIGHUP, reloading configuration files - 2023-01-05 15:09:57.580 UTC [39] LOG: parameter "synchronous_standby_names" changed to "coord3" - 2023-01-05 15:09:59,637 INFO: Synchronous standby status assigned to ['coord3'] - 2023-01-05 15:09:59,638 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.27.0.2', 5432, 1)) - 2023-01-05 15:09:59.690 UTC [67] LOG: standby "coord3" is now a synchronous standby with priority 1 - 2023-01-05 15:09:59.690 UTC [67] STATEMENT: START_REPLICATION SLOT "coord3" 0/3000000 TIMELINE 1 - 2023-01-05 15:09:59,694 INFO: no action. I am (coord1), the leader with the lock - 2023-01-05 15:09:59,704 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.27.0.8', 5432, 2)) - 2023-01-05 15:10:07,625 INFO: no action. I am (coord1), the leader with the lock - 2023-01-05 15:10:17,579 INFO: no action. I am (coord1), the leader with the lock + 2023-11-21 09:36:28.435 UTC [52] LOG: received SIGHUP, reloading configuration files + 2023-11-21 09:36:28.436 UTC [52] LOG: parameter "synchronous_standby_names" changed to "coord3" + 2023-11-21 09:36:28.641 UTC [83] LOG: standby "coord3" is now a synchronous standby with priority 1 + 2023-11-21 09:36:28.641 UTC [83] STATEMENT: START_REPLICATION SLOT "coord3" 0/3000000 TIMELINE 1 + 2023-11-21 09:36:30,582 INFO: Synchronous standby status assigned to ['coord3'] + 2023-11-21 09:36:30,626 INFO: no action. I am (coord1), the leader with the lock + 2023-11-21 09:36:38,250 INFO: no action. I am (coord1), the leader with the lock + ... $ docker exec -ti demo-haproxy bash postgres@haproxy:~$ etcdctl member list - 1bab629f01fa9065, started, etcd3, http://etcd3:2380, http://172.27.0.10:2379 - 8ecb6af518d241cc, started, etcd2, http://etcd2:2380, http://172.27.0.4:2379 - b2e169fcb8a34028, started, etcd1, http://etcd1:2380, http://172.27.0.7:2379 + 2b28411e74c0c281, started, etcd3, http://etcd3:2380, http://172.30.0.4:2379 + 6c70137d27cfa6c1, started, etcd2, http://etcd2:2380, http://172.30.0.5:2379 + a28f9a70ebf21304, started, etcd1, http://etcd1:2380, http://172.30.0.6:2379 postgres@haproxy:~$ etcdctl get --keys-only --prefix /service/demo /service/demo/0/config @@ -229,7 +229,7 @@ Example session: postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -d citus Password for user postgres: postgres - psql (15.1 (Debian 15.1-1.pgdg110+1)) + psql (15.5 (Debian 15.5-1.pgdg120+1)) SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, compression: off) Type "help" for help. @@ -240,67 +240,67 @@ Example session: (1 row) citus=# table pg_dist_node; - nodeid | groupid | nodename | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards + nodeid | groupid | nodename | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards --------+---------+------------+----------+----------+-------------+----------+----------+-------------+----------------+------------------ - 1 | 0 | 172.27.0.6 | 5432 | default | t | t | primary | default | t | f - 2 | 1 | 172.27.0.2 | 5432 | default | t | t | primary | default | t | t - 3 | 2 | 172.27.0.8 | 5432 | default | t | t | primary | default | t | t + 1 | 0 | 172.30.0.3 | 5432 | default | t | t | primary | default | t | f + 2 | 1 | 172.30.0.7 | 5432 | default | t | t | primary | default | t | t + 3 | 2 | 172.30.0.8 | 5432 | default | t | t | primary | default | t | t (3 rows) citus=# \q postgres@haproxy:~$ patronictl list - + Citus cluster: demo ----------+--------------+---------+----+-----------+ - | Group | Member | Host | Role | State | TL | Lag in MB | - +-------+---------+-------------+--------------+---------+----+-----------+ - | 0 | coord1 | 172.27.0.6 | Leader | running | 1 | | - | 0 | coord2 | 172.27.0.5 | Replica | running | 1 | 0 | - | 0 | coord3 | 172.27.0.9 | Sync Standby | running | 1 | 0 | - | 1 | work1-1 | 172.27.0.2 | Leader | running | 1 | | - | 1 | work1-2 | 172.27.0.12 | Sync Standby | running | 1 | 0 | - | 2 | work2-1 | 172.27.0.11 | Sync Standby | running | 1 | 0 | - | 2 | work2-2 | 172.27.0.8 | Leader | running | 1 | | - +-------+---------+-------------+--------------+---------+----+-----------+ + + Citus cluster: demo ----------+--------------+-----------+----+-----------+ + | Group | Member | Host | Role | State | TL | Lag in MB | + +-------+---------+-------------+--------------+-----------+----+-----------+ + | 0 | coord1 | 172.30.0.3 | Leader | running | 1 | | + | 0 | coord2 | 172.30.0.12 | Replica | streaming | 1 | 0 | + | 0 | coord3 | 172.30.0.2 | Sync Standby | streaming | 1 | 0 | + | 1 | work1-1 | 172.30.0.7 | Leader | running | 1 | | + | 1 | work1-2 | 172.30.0.10 | Sync Standby | streaming | 1 | 0 | + | 2 | work2-1 | 172.30.0.8 | Leader | running | 1 | | + | 2 | work2-2 | 172.30.0.11 | Sync Standby | streaming | 1 | 0 | + +-------+---------+-------------+--------------+-----------+----+-----------+ + postgres@haproxy:~$ patronictl switchover --group 2 --force Current cluster topology - + Citus cluster: demo (group: 2, 7185185529556963355) +-----------+ - | Member | Host | Role | State | TL | Lag in MB | - +---------+-------------+--------------+---------+----+-----------+ - | work2-1 | 172.27.0.11 | Sync Standby | running | 1 | 0 | - | work2-2 | 172.27.0.8 | Leader | running | 1 | | - +---------+-------------+--------------+---------+----+-----------+ - 2023-01-05 15:29:29.54204 Successfully switched over to "work2-1" - + Citus cluster: demo (group: 2, 7185185529556963355) -------+ + + Citus cluster: demo (group: 2, 7303846899271086103) --+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +---------+-------------+--------------+-----------+----+-----------+ + | work2-1 | 172.30.0.8 | Leader | running | 1 | | + | work2-2 | 172.30.0.11 | Sync Standby | streaming | 1 | 0 | + +---------+-------------+--------------+-----------+----+-----------+ + 2023-11-21 09:44:15.83849 Successfully switched over to "work2-2" + + Citus cluster: demo (group: 2, 7303846899271086103) -------+ | Member | Host | Role | State | TL | Lag in MB | +---------+-------------+---------+---------+----+-----------+ - | work2-1 | 172.27.0.11 | Leader | running | 1 | | - | work2-2 | 172.27.0.8 | Replica | stopped | | unknown | + | work2-1 | 172.30.0.8 | Replica | stopped | | unknown | + | work2-2 | 172.30.0.11 | Leader | running | 1 | | +---------+-------------+---------+---------+----+-----------+ postgres@haproxy:~$ patronictl list - + Citus cluster: demo ----------+--------------+---------+----+-----------+ - | Group | Member | Host | Role | State | TL | Lag in MB | - +-------+---------+-------------+--------------+---------+----+-----------+ - | 0 | coord1 | 172.27.0.6 | Leader | running | 1 | | - | 0 | coord2 | 172.27.0.5 | Replica | running | 1 | 0 | - | 0 | coord3 | 172.27.0.9 | Sync Standby | running | 1 | 0 | - | 1 | work1-1 | 172.27.0.2 | Leader | running | 1 | | - | 1 | work1-2 | 172.27.0.12 | Sync Standby | running | 1 | 0 | - | 2 | work2-1 | 172.27.0.11 | Leader | running | 2 | | - | 2 | work2-2 | 172.27.0.8 | Sync Standby | running | 2 | 0 | - +-------+---------+-------------+--------------+---------+----+-----------+ + + Citus cluster: demo ----------+--------------+-----------+----+-----------+ + | Group | Member | Host | Role | State | TL | Lag in MB | + +-------+---------+-------------+--------------+-----------+----+-----------+ + | 0 | coord1 | 172.30.0.3 | Leader | running | 1 | | + | 0 | coord2 | 172.30.0.12 | Replica | streaming | 1 | 0 | + | 0 | coord3 | 172.30.0.2 | Sync Standby | streaming | 1 | 0 | + | 1 | work1-1 | 172.30.0.7 | Leader | running | 1 | | + | 1 | work1-2 | 172.30.0.10 | Sync Standby | streaming | 1 | 0 | + | 2 | work2-1 | 172.30.0.8 | Sync Standby | streaming | 2 | 0 | + | 2 | work2-2 | 172.30.0.11 | Leader | running | 2 | | + +-------+---------+-------------+--------------+-----------+----+-----------+ postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -d citus - Password for user postgres: postgres - psql (15.1 (Debian 15.1-1.pgdg110+1)) + psql (15.5 (Debian 15.5-1.pgdg120+1)) SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, compression: off) Type "help" for help. citus=# table pg_dist_node; - nodeid | groupid | nodename | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards + nodeid | groupid | nodename | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards --------+---------+-------------+----------+----------+-------------+----------+----------+-------------+----------------+------------------ - 1 | 0 | 172.27.0.6 | 5432 | default | t | t | primary | default | t | f - 3 | 2 | 172.27.0.11 | 5432 | default | t | t | primary | default | t | t - 2 | 1 | 172.27.0.2 | 5432 | default | t | t | primary | default | t | t + 1 | 0 | 172.30.0.3 | 5432 | default | t | t | primary | default | t | f + 3 | 2 | 172.30.0.11 | 5432 | default | t | t | primary | default | t | t + 2 | 1 | 172.30.0.7 | 5432 | default | t | t | primary | default | t | t (3 rows) From 91327f943c92faa4bde478494e8c7338138badbc Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 23 Nov 2023 17:04:23 +0100 Subject: [PATCH 18/19] Factor out dynamic class finder/loader to a dedicated file (#2954) It could be reused to do the same for MPP modules/classes. Ref: #2940 and #2950 --- patroni/dcs/__init__.py | 72 ++++------------------------- patroni/dynamic_loader.py | 96 +++++++++++++++++++++++++++++++++++++++ tests/test_ctl.py | 2 +- 3 files changed, 107 insertions(+), 63 deletions(-) create mode 100644 patroni/dynamic_loader.py diff --git a/patroni/dcs/__init__.py b/patroni/dcs/__init__.py index 9c2f7335..b2538e1a 100644 --- a/patroni/dcs/__init__.py +++ b/patroni/dcs/__init__.py @@ -1,26 +1,21 @@ """Abstract classes for Distributed Configuration Store.""" import abc import datetime -import importlib -import inspect import json import logging -import os -import pkgutil import re -import sys import time from collections import defaultdict from copy import deepcopy from random import randint from threading import Event, Lock -from types import ModuleType -from typing import Any, Callable, Collection, Dict, List, NamedTuple, Optional, Set, Tuple, Union, TYPE_CHECKING, \ - Type, Iterator +from typing import Any, Callable, Collection, Dict, Iterator, List, \ + NamedTuple, Optional, Tuple, Type, TYPE_CHECKING, Union from urllib.parse import urlparse, urlunparse, parse_qsl import dateutil.parser +from ..dynamic_loader import iter_classes, iter_modules from ..exceptions import PatroniFatalException from ..utils import deep_compare, uri from ..tags import Tags @@ -87,28 +82,9 @@ def parse_connection_string(value: str) -> Tuple[str, Union[str, None]]: def dcs_modules() -> List[str]: """Get names of DCS modules, depending on execution environment. - .. note:: - If being packaged with PyInstaller, modules aren't discoverable dynamically by scanning source directory because - :class:`importlib.machinery.FrozenImporter` doesn't implement :func:`iter_modules`. But it is still possible to - find all potential DCS modules by iterating through ``toc``, which contains list of all "frozen" resources. - :returns: list of known module names with absolute python module path namespace, e.g. ``patroni.dcs.etcd``. """ - dcs_dirname = os.path.dirname(__file__) - module_prefix = __package__ + '.' - - if getattr(sys, 'frozen', False): - toc: Set[str] = set() - # dcs_dirname may contain a dot, which causes pkgutil.iter_importers() - # to misinterpret the path as a package name. This can be avoided - # altogether by not passing a path at all, because PyInstaller's - # FrozenImporter is a singleton and registered as top-level finder. - for importer in pkgutil.iter_importers(): - if hasattr(importer, 'toc'): - toc |= getattr(importer, 'toc') - return [module for module in toc if module.startswith(module_prefix) and module.count('.') == 2] - - return [module_prefix + name for _, name, is_pkg in pkgutil.iter_modules([dcs_dirname]) if not is_pkg] + return iter_modules(__package__) def iter_dcs_classes( @@ -122,44 +98,16 @@ def iter_dcs_classes( :param config: configuration information with possible DCS names as keys. If given, only attempt to import DCS modules defined in the configuration. Else, if ``None``, attempt to import any supported DCS module. - :yields: a tuple containing the module ``name`` and the imported DCS class object. + :returns: an iterator of tuples, each containing the module ``name`` and the imported DCS class object. """ - for mod_name in dcs_modules(): - name = mod_name.rpartition('.')[2] - if config is None or name in config: - - try: - module = importlib.import_module(mod_name) - dcs_module = find_dcs_class_in_module(module) - if dcs_module: - yield name, dcs_module - - except ImportError: - logger.log(logging.DEBUG if config is not None else logging.INFO, - 'Failed to import %s', mod_name) - - -def find_dcs_class_in_module(module: ModuleType) -> Optional[Type['AbstractDCS']]: - """Try to find the implementation of :class:`AbstractDCS` interface in *module* matching the *module* name. - - :param module: Imported DCS module. - - :returns: class with a name matching the name of *module* that implements :class:`AbstractDCS` or ``None`` if not - found. - """ - module_name = module.__name__.rpartition('.')[2] - return next( - (obj for obj_name, obj in module.__dict__.items() - if (obj_name.lower() == module_name - and inspect.isclass(obj) and issubclass(obj, AbstractDCS))), - None) + return iter_classes(__package__, AbstractDCS, config) def get_dcs(config: Union['Config', Dict[str, Any]]) -> 'AbstractDCS': """Attempt to load a Distributed Configuration Store from known available implementations. .. note:: - Using the list of available DCS classes returned by :func:`iter_dcs_classes` attempt to dynamically + Using the list of available DCS classes returned by :func:`iter_classes` attempt to dynamically instantiate the class that implements a DCS using the abstract class :class:`AbstractDCS`. Basic top-level configuration parameters retrieved from *config* are propagated to the DCS specific config @@ -185,9 +133,9 @@ def get_dcs(config: Union['Config', Dict[str, Any]]) -> 'AbstractDCS': config[name].update(config['citus']) return dcs_class(config[name]) - raise PatroniFatalException( - f"Can not find suitable configuration of distributed configuration store\n" - f"Available implementations: {', '.join(sorted([n for n, _ in iter_dcs_classes()]))}") + available_implementations = ', '.join(sorted([n for n, _ in iter_dcs_classes()])) + raise PatroniFatalException("Can not find suitable configuration of distributed configuration store\n" + f"Available implementations: {available_implementations}") _Version = Union[int, str] diff --git a/patroni/dynamic_loader.py b/patroni/dynamic_loader.py new file mode 100644 index 00000000..6c207349 --- /dev/null +++ b/patroni/dynamic_loader.py @@ -0,0 +1,96 @@ +"""Helper functions to search for implementations of specific abstract interface in a package.""" +import importlib +import inspect +import logging +import os +import pkgutil +import sys +from types import ModuleType + +from typing import Any, Dict, Iterator, List, Optional, Set, Tuple, TYPE_CHECKING, Type, TypeVar, Union + +if TYPE_CHECKING: # pragma: no cover + from .config import Config + +logger = logging.getLogger(__name__) + + +def iter_modules(package: str) -> List[str]: + """Get names of modules from *package*, depending on execution environment. + + .. note:: + If being packaged with PyInstaller, modules aren't discoverable dynamically by scanning source directory because + :class:`importlib.machinery.FrozenImporter` doesn't implement :func:`iter_modules`. But it is still possible to + find all potential modules by iterating through ``toc``, which contains list of all "frozen" resources. + + :param package: a package name to search modules in, e.g. ``patroni.dcs``. + + :returns: list of known module names with absolute python module path namespace, e.g. ``patroni.dcs.etcd``. + """ + module_prefix = package + '.' + + if getattr(sys, 'frozen', False): + toc: Set[str] = set() + # dirname may contain a few dots, which causes pkgutil.iter_importers() + # to misinterpret the path as a package name. This can be avoided + # altogether by not passing a path at all, because PyInstaller's + # FrozenImporter is a singleton and registered as top-level finder. + for importer in pkgutil.iter_importers(): + if hasattr(importer, 'toc'): + toc |= getattr(importer, 'toc') + dots = module_prefix.count('.') # search for modules only on the same level + return [module for module in toc if module.startswith(module_prefix) and module.count('.') == dots] + + # here we are making an assumption that the package which is calling this function is already imported + pkg_file = sys.modules[package].__file__ + if TYPE_CHECKING: # pragma: no cover + assert isinstance(pkg_file, str) + return [name for _, name, is_pkg in pkgutil.iter_modules([os.path.dirname(pkg_file)], module_prefix) if not is_pkg] + + +ClassType = TypeVar("ClassType") + + +def find_class_in_module(module: ModuleType, cls_type: Type[ClassType]) -> Optional[Type[ClassType]]: + """Try to find the implementation of *cls_type* class interface in *module* matching the *module* name. + + :param module: imported module. + :param cls_type: a class type we are looking for. + + :returns: class with a name matching the name of *module* that implements *cls_type* or ``None`` if not found. + """ + module_name = module.__name__.rpartition('.')[2] + return next( + (obj for obj_name, obj in module.__dict__.items() + if (obj_name.lower() == module_name + and inspect.isclass(obj) and issubclass(obj, cls_type))), + None) + + +def iter_classes( + package: str, cls_type: Type[ClassType], + config: Optional[Union['Config', Dict[str, Any]]] = None +) -> Iterator[Tuple[str, Type[ClassType]]]: + """Attempt to import modules and find implementations of *cls_type* that are present in the given configuration. + + .. note:: + If a module successfully imports we can assume that all its requirements are installed. + + :param package: a package name to search modules in, e.g. ``patroni.dcs``. + :param cls_type: a class type we are looking for. + :param config: configuration information with possible module names as keys. If given, only attempt to import + modules defined in the configuration. Else, if ``None``, attempt to import any supported module. + + :yields: a tuple containing the module ``name`` and the imported class object. + """ + for mod_name in iter_modules(package): + name = mod_name.rpartition('.')[2] + if config is None or name in config: + try: + module = importlib.import_module(mod_name) + module_cls = find_class_in_module(module, cls_type) + if module_cls: + yield name, module_cls + except ImportError: + logger.log(logging.DEBUG if config is not None else logging.INFO, + 'Failed to import %s', mod_name) diff --git a/tests/test_ctl.py b/tests/test_ctl.py index bacd2d60..96c36c16 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -238,7 +238,7 @@ class TestCtl(unittest.TestCase): result = self.runner.invoke(ctl, ['failover', 'dummy', '--group', '0', '--candidate', 'async'], input='N') self.assertEqual(result.exit_code, 1) - @patch('patroni.dcs.dcs_modules', Mock(return_value=['patroni.dcs.dummy', 'patroni.dcs.etcd'])) + @patch('patroni.dynamic_loader.iter_modules', Mock(return_value=['patroni.dcs.dummy', 'patroni.dcs.etcd'])) def test_get_dcs(self): with click.Context(click.Command('list')) as ctx: ctx.obj = {'__config': {'dummy': {}}} From 193c73f6b80c42b316d790f9b40321da2ee55a31 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 24 Nov 2023 09:26:05 +0100 Subject: [PATCH 19/19] Make GlobalConfig really global (#2935) 1. extract `GlobalConfig` class to its own module 2. make the module instantiate the `GlobalConfig` object on load and replace sys.modules with the this instance 3. don't pass `GlobalConfig` object around, but use `patroni.global_config` module everywhere. 4. move `ignore_slots_matchers`, `max_timelines_history`, and `permanent_slots` from `ClusterConfig` to `GlobalConfig`. 5. add `use_slots` property to global_config and remove duplicated code from `Cluster` and `Postgresql.ConfigHandler`. Besides that improve readability of couple of checks in ha.py and formatting of `/config` key when saved from patronictl. --- patroni/api.py | 27 ++-- patroni/config.py | 162 +---------------------- patroni/ctl.py | 21 +-- patroni/dcs/__init__.py | 62 ++------- patroni/global_config.py | 227 +++++++++++++++++++++++++++++++++ patroni/ha.py | 48 ++++--- patroni/postgresql/__init__.py | 25 +--- patroni/postgresql/config.py | 7 +- patroni/postgresql/slots.py | 10 +- patroni/postgresql/sync.py | 8 +- patroni/utils.py | 15 +-- tests/test_api.py | 29 ++--- tests/test_config.py | 10 +- tests/test_ctl.py | 11 +- tests/test_ha.py | 32 +++-- tests/test_patroni.py | 1 + tests/test_postgresql.py | 14 +- tests/test_slots.py | 10 +- tests/test_sync.py | 7 +- 19 files changed, 372 insertions(+), 354 deletions(-) create mode 100644 patroni/global_config.py diff --git a/patroni/api.py b/patroni/api.py index 6a89b1ba..2adc5f98 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -26,7 +26,7 @@ from urllib.parse import urlparse, parse_qs from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union -from . import psycopg +from . import global_config, psycopg from .__main__ import Patroni from .dcs import Cluster from .exceptions import PostgresConnectionException, PostgresException @@ -290,7 +290,7 @@ class RestApiHandler(BaseHTTPRequestHandler): patroni = self.server.patroni cluster = patroni.dcs.cluster - global_config = patroni.config.get_global_config(cluster) + config = global_config.from_cluster(cluster) leader_optime = cluster and cluster.last_lsn or 0 replayed_location = response.get('xlog', {}).get('replayed_location', 0) @@ -308,7 +308,7 @@ class RestApiHandler(BaseHTTPRequestHandler): standby_leader_status_code = 200 if response.get('role') == 'standby_leader' else 503 elif patroni.ha.is_leader(): leader_status_code = 200 - if global_config.is_standby_cluster: + if config.is_standby_cluster: primary_status_code = replica_status_code = 503 standby_leader_status_code = 200 if response.get('role') in ('replica', 'standby_leader') else 503 else: @@ -452,9 +452,8 @@ class RestApiHandler(BaseHTTPRequestHandler): HTTP status ``200`` and the JSON representation of the cluster topology. """ cluster = self.server.patroni.dcs.get_cluster() - global_config = self.server.patroni.config.get_global_config(cluster) - response = cluster_as_json(cluster, global_config) + response = cluster_as_json(cluster) response['scope'] = self.server.patroni.postgresql.scope self._write_json_response(200, response) @@ -864,7 +863,7 @@ class RestApiHandler(BaseHTTPRequestHandler): if request: logger.debug("received restart request: {0}".format(request)) - if self.server.patroni.config.get_global_config(cluster).is_paused and 'schedule' in request: + if global_config.from_cluster(cluster).is_paused and 'schedule' in request: self.write_response(status_code, "Can't schedule restart in the paused state") return @@ -1033,7 +1032,7 @@ class RestApiHandler(BaseHTTPRequestHandler): :returns: a string with the error message or ``None`` if good nodes are found. """ - is_synchronous_mode = self.server.patroni.config.get_global_config(cluster).is_synchronous_mode + is_synchronous_mode = global_config.from_cluster(cluster).is_synchronous_mode if leader and (not cluster.leader or cluster.leader.name != leader): return 'leader name does not match' if candidate: @@ -1091,7 +1090,7 @@ class RestApiHandler(BaseHTTPRequestHandler): candidate = request.get('candidate') or request.get('member') scheduled_at = request.get('scheduled_at') cluster = self.server.patroni.dcs.get_cluster() - global_config = self.server.patroni.config.get_global_config(cluster) + config = global_config.from_cluster(cluster) logger.info("received %s request with leader=%s candidate=%s scheduled_at=%s", action, leader, candidate, scheduled_at) @@ -1104,12 +1103,12 @@ class RestApiHandler(BaseHTTPRequestHandler): if not data and scheduled_at: if action == 'failover': data = "Failover can't be scheduled" - elif global_config.is_paused: + elif config.is_paused: data = "Can't schedule switchover in the paused state" else: (status_code, data, scheduled_at) = self.parse_schedule(scheduled_at, action) - if not data and global_config.is_paused and not candidate: + if not data and config.is_paused and not candidate: data = 'Switchover is possible only to a specific candidate in a paused state' if action == 'failover' and leader: @@ -1260,7 +1259,7 @@ class RestApiHandler(BaseHTTPRequestHandler): """ postgresql = self.server.patroni.postgresql cluster = self.server.patroni.dcs.cluster - global_config = self.server.patroni.config.get_global_config(cluster) + config = global_config.from_cluster(cluster) try: if postgresql.state not in ('running', 'restarting', 'starting'): @@ -1291,10 +1290,10 @@ class RestApiHandler(BaseHTTPRequestHandler): }) } - if result['role'] == 'replica' and global_config.is_standby_cluster: + if result['role'] == 'replica' and config.is_standby_cluster: result['role'] = postgresql.role - if result['role'] == 'replica' and global_config.is_synchronous_mode\ + if result['role'] == 'replica' and config.is_synchronous_mode\ and cluster and cluster.sync.matches(postgresql.name): result['sync_standby'] = True @@ -1319,7 +1318,7 @@ class RestApiHandler(BaseHTTPRequestHandler): state = 'unknown' result: Dict[str, Any] = {'state': state, 'role': postgresql.role} - if global_config.is_paused: + if config.is_paused: result['pause'] = True if not cluster or cluster.is_unlocked(): result['cluster_unlocked'] = True diff --git a/patroni/config.py b/patroni/config.py index 00dba9d9..e523bc08 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -12,7 +12,7 @@ from typing import Any, Callable, Collection, Dict, List, Optional, Union, TYPE_ from . import PATRONI_ENV_PREFIX from .collections import CaseInsensitiveDict -from .dcs import ClusterConfig, Cluster +from .dcs import ClusterConfig from .exceptions import ConfigParseError from .file_perm import pg_perm from .postgresql.config import ConfigHandler @@ -54,154 +54,6 @@ def default_validator(conf: Dict[str, Any]) -> List[str]: return [] -class GlobalConfig(object): - """A class that wraps global configuration and provides convenient methods to access/check values. - - It is instantiated either by calling :func:`get_global_config` or :meth:`Config.get_global_config`, which picks - either a configuration from provided :class:`Cluster` object (the most up-to-date) or from the - local cache if :class:`ClusterConfig` is not initialized or doesn't have a valid config. - """ - - def __init__(self, config: Dict[str, Any]) -> None: - """Initialize :class:`GlobalConfig` object with given *config*. - - :param config: current configuration either from - :class:`ClusterConfig` or from :func:`Config.dynamic_configuration`. - """ - self.__config = config - - def get(self, name: str) -> Any: - """Gets global configuration value by *name*. - - :param name: parameter name. - - :returns: configuration value or ``None`` if it is missing. - """ - return self.__config.get(name) - - def check_mode(self, mode: str) -> bool: - """Checks whether the certain parameter is enabled. - - :param mode: parameter name, e.g. ``synchronous_mode``, ``failsafe_mode``, ``pause``, ``check_timeline``, and - so on. - - :returns: ``True`` if parameter *mode* is enabled in the global configuration. - """ - return bool(parse_bool(self.__config.get(mode))) - - @property - def is_paused(self) -> bool: - """``True`` if cluster is in maintenance mode.""" - return self.check_mode('pause') - - @property - def is_synchronous_mode(self) -> bool: - """``True`` if synchronous replication is requested and it is not a standby cluster config.""" - return self.check_mode('synchronous_mode') and not self.is_standby_cluster - - @property - def is_synchronous_mode_strict(self) -> bool: - """``True`` if at least one synchronous node is required.""" - return self.check_mode('synchronous_mode_strict') - - def get_standby_cluster_config(self) -> Union[Dict[str, Any], Any]: - """Get ``standby_cluster`` configuration. - - :returns: a copy of ``standby_cluster`` configuration. - """ - return deepcopy(self.get('standby_cluster')) - - @property - def is_standby_cluster(self) -> bool: - """``True`` if global configuration has a valid ``standby_cluster`` section.""" - config = self.get_standby_cluster_config() - return isinstance(config, dict) and\ - bool(config.get('host') or config.get('port') or config.get('restore_command')) - - def get_int(self, name: str, default: int = 0) -> int: - """Gets current value of *name* from the global configuration and try to return it as :class:`int`. - - :param name: name of the parameter. - :param default: default value if *name* is not in the configuration or invalid. - - :returns: currently configured value of *name* from the global configuration or *default* if it is not set or - invalid. - """ - ret = parse_int(self.get(name)) - return default if ret is None else ret - - @property - def min_synchronous_nodes(self) -> int: - """The minimal number of synchronous nodes based on whether ``synchronous_mode_strict`` is enabled or not.""" - return 1 if self.is_synchronous_mode_strict else 0 - - @property - def synchronous_node_count(self) -> int: - """Currently configured value of ``synchronous_node_count`` from the global configuration. - - Assume ``1`` if it is not set or invalid. - """ - return max(self.get_int('synchronous_node_count', 1), self.min_synchronous_nodes) - - @property - def maximum_lag_on_failover(self) -> int: - """Currently configured value of ``maximum_lag_on_failover`` from the global configuration. - - Assume ``1048576`` if it is not set or invalid. - """ - return self.get_int('maximum_lag_on_failover', 1048576) - - @property - def maximum_lag_on_syncnode(self) -> int: - """Currently configured value of ``maximum_lag_on_syncnode`` from the global configuration. - - Assume ``-1`` if it is not set or invalid. - """ - return self.get_int('maximum_lag_on_syncnode', -1) - - @property - def primary_start_timeout(self) -> int: - """Currently configured value of ``primary_start_timeout`` from the global configuration. - - Assume ``300`` if it is not set or invalid. - - .. note:: - ``master_start_timeout`` is still supported to keep backward compatibility. - """ - default = 300 - return self.get_int('primary_start_timeout', default)\ - if 'primary_start_timeout' in self.__config else self.get_int('master_start_timeout', default) - - @property - def primary_stop_timeout(self) -> int: - """Currently configured value of ``primary_stop_timeout`` from the global configuration. - - Assume ``0`` if it is not set or invalid. - - .. note:: - ``master_stop_timeout`` is still supported to keep backward compatibility. - """ - default = 0 - return self.get_int('primary_stop_timeout', default)\ - if 'primary_stop_timeout' in self.__config else self.get_int('master_stop_timeout', default) - - -def get_global_config(cluster: Optional[Cluster], default: Optional[Dict[str, Any]] = None) -> GlobalConfig: - """Instantiates :class:`GlobalConfig` based on the input. - - :param cluster: the currently known cluster state from DCS. - :param default: default configuration, which will be used if there is no valid *cluster.config*. - - :returns: :class:`GlobalConfig` object. - """ - # Try to protect from the case when DCS was wiped out - if cluster and cluster.config and cluster.config.modify_version: - config = cluster.config.data - else: - config = default or {} - return GlobalConfig(deepcopy(config)) - - class Config(object): """Handle Patroni configuration. @@ -949,18 +801,6 @@ class Config(object): """ return deepcopy(self.__effective_configuration) - def get_global_config(self, cluster: Optional[Cluster]) -> GlobalConfig: - """Instantiate :class:`GlobalConfig` based on input. - - Use the configuration from provided *cluster* (the most up-to-date) or from the - local cache if *cluster.config* is not initialized or doesn't have a valid config. - - :param cluster: the currently known cluster state from DCS. - - :returns: :class:`GlobalConfig` object. - """ - return get_global_config(cluster, self._dynamic_configuration) - def _validate_failover_tags(self) -> None: """Check ``nofailover``/``failover_priority`` config and warn user if it's contradictory. diff --git a/patroni/ctl.py b/patroni/ctl.py index 2d92457a..13959896 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -46,7 +46,8 @@ try: except ImportError: # pragma: no cover from cdiff import markup_to_pager, PatchStream # pyright: ignore [reportMissingModuleSource] -from .config import Config, get_global_config +from . import global_config +from .config import Config from .dcs import get_dcs as _get_dcs, AbstractDCS, Cluster, Member from .exceptions import PatroniException from .postgresql.misc import postgres_version_to_int @@ -1026,7 +1027,7 @@ def reload(cluster_name: str, member_names: List[str], group: Optional[int], for if r.status == 200: click.echo('No changes to apply on member {0}'.format(member.name)) elif r.status == 202: - config = get_global_config(cluster) + config = global_config.from_cluster(cluster) click.echo('Reload request received for member {0} and will be processed within {1} seconds'.format( member.name, config.get('loop_wait') or dcs.loop_wait) ) @@ -1105,7 +1106,7 @@ def restart(cluster_name: str, group: Optional[int], member_names: List[str], content['postgres_version'] = version if scheduled_at: - if get_global_config(cluster).is_paused: + if global_config.from_cluster(cluster).is_paused: raise PatroniCtlException("Can't schedule restart in the paused state") content['schedule'] = scheduled_at.isoformat() @@ -1228,7 +1229,7 @@ def _do_failover_or_switchover(action: str, cluster_name: str, group: Optional[i dcs = get_dcs(cluster_name, group) cluster = dcs.get_cluster() - global_config = get_global_config(cluster) + config = global_config.from_cluster(cluster) # leader has to be be defined for switchover only if action == 'switchover': @@ -1239,7 +1240,7 @@ def _do_failover_or_switchover(action: str, cluster_name: str, group: Optional[i if force: leader = cluster.leader.name else: - prompt = 'Standby Leader' if global_config.is_standby_cluster else 'Primary' + prompt = 'Standby Leader' if config.is_standby_cluster else 'Primary' leader = click.prompt(prompt, type=str, default=(cluster.leader and cluster.leader.name)) if cluster.leader.name != leader: @@ -1268,7 +1269,7 @@ def _do_failover_or_switchover(action: str, cluster_name: str, group: Optional[i if all((not force, action == 'failover', - global_config.is_synchronous_mode, + config.is_synchronous_mode, not cluster.sync.is_empty, not cluster.sync.matches(candidate, True))): if click.confirm(f'Are you sure you want to failover to the asynchronous node {candidate}'): @@ -1285,7 +1286,7 @@ def _do_failover_or_switchover(action: str, cluster_name: str, group: Optional[i scheduled_at = parse_scheduled(scheduled) if scheduled_at: - if global_config.is_paused: + if config.is_paused: raise PatroniCtlException("Can't schedule switchover in the paused state") scheduled_at_str = scheduled_at.isoformat() @@ -1739,7 +1740,7 @@ def wait_until_pause_is_applied(dcs: AbstractDCS, paused: bool, old_cluster: Clu :param old_cluster: original cluster information before pause or unpause has been requested. Used to report which nodes are still pending to have ``pause`` equal *paused* at a given point in time. """ - config = get_global_config(old_cluster) + config = global_config.from_cluster(old_cluster) click.echo("'{0}' request sent, waiting until it is recognized by all nodes".format(paused and 'pause' or 'resume')) old = {m.name: m.version for m in old_cluster.members if m.api_url} @@ -1777,7 +1778,7 @@ def toggle_pause(cluster_name: str, group: Optional[int], paused: bool, wait: bo """ dcs = get_dcs(cluster_name, group) cluster = dcs.get_cluster() - if get_global_config(cluster).is_paused == paused: + if global_config.from_cluster(cluster).is_paused == paused: raise PatroniCtlException('Cluster is {0} paused'.format(paused and 'already' or 'not')) for member in get_all_members_leader_first(cluster): @@ -2122,7 +2123,7 @@ def edit_config(cluster_name: str, group: Optional[int], force: bool, quiet: boo return if force or click.confirm('Apply these changes?'): - if not dcs.set_config_value(json.dumps(changed_data), cluster.config.version): + if not dcs.set_config_value(json.dumps(changed_data, separators=(',', ':')), cluster.config.version): raise PatroniCtlException("Config modification aborted due to concurrent changes") click.echo("Configuration changed") diff --git a/patroni/dcs/__init__.py b/patroni/dcs/__init__.py index b2538e1a..9298e3fc 100644 --- a/patroni/dcs/__init__.py +++ b/patroni/dcs/__init__.py @@ -15,6 +15,7 @@ from urllib.parse import urlparse, urlunparse, parse_qsl import dateutil.parser +from .. import global_config from ..dynamic_loader import iter_classes, iter_modules from ..exceptions import PatroniFatalException from ..utils import deep_compare, uri @@ -538,24 +539,6 @@ class ClusterConfig(NamedTuple): modify_version = 0 return ClusterConfig(version, data, version if modify_version is None else modify_version) - @property - def permanent_slots(self) -> Dict[str, Any]: - """Dictionary of permanent slots information looked up from :attr:`~ClusterConfig.data`.""" - return (self.data.get('permanent_replication_slots') - or self.data.get('permanent_slots') - or self.data.get('slots') - or {}) - - @property - def ignore_slots_matchers(self) -> List[Dict[str, Any]]: - """The value for ``ignore_slots`` from :attr:`~ClusterConfig.data` if defined or an empty list.""" - return self.data.get('ignore_slots') or [] - - @property - def max_timelines_history(self) -> int: - """The value for ``max_timelines_history`` from :attr:`~ClusterConfig.data` if defined or ``0``.""" - return self.data.get('max_timelines_history', 0) - class SyncState(NamedTuple): """Immutable object (namedtuple) which represents last observed synchronous replication state. @@ -944,7 +927,7 @@ class Cluster(NamedTuple('Cluster', @property def __permanent_slots(self) -> Dict[str, Union[Dict[str, Any], Any]]: """Dictionary of permanent replication slots with their known LSN.""" - ret: Dict[str, Union[Dict[str, Any], Any]] = deepcopy(self.config.permanent_slots if self.config else {}) + ret: Dict[str, Union[Dict[str, Any], Any]] = global_config.permanent_slots members: Dict[str, int] = {slot_name_from_member_name(m.name): m.lsn or 0 for m in self.members} slots: Dict[str, int] = {k: parse_int(v) or 0 for k, v in (self.slots or {}).items()} @@ -973,13 +956,8 @@ class Cluster(NamedTuple('Cluster', """Dictionary of permanent ``logical`` replication slots.""" return {name: value for name, value in self.__permanent_slots.items() if self.is_logical_slot(value)} - @property - def use_slots(self) -> bool: - """``True`` if cluster is configured to use replication slots.""" - return bool(self.config and (self.config.data.get('postgresql') or {}).get('use_slots', True)) - def get_replication_slots(self, my_name: str, role: str, nofailover: bool, major_version: int, *, - is_standby_cluster: bool = False, show_error: bool = False) -> Dict[str, Dict[str, Any]]: + show_error: bool = False) -> Dict[str, Dict[str, Any]]: """Lookup configured slot names in the DCS, report issues found and merge with permanent slots. Will log an error if: @@ -990,15 +968,12 @@ class Cluster(NamedTuple('Cluster', :param role: role of this node. :param nofailover: ``True`` if this node is tagged to not be a failover candidate. :param major_version: postgresql major version. - :param is_standby_cluster: ``True`` if it is known that this is a standby cluster. We pass the value from - the outside because we want to protect from the ``/config`` key removal. :param show_error: if ``True`` report error if any disabled logical slots or conflicting slot names are found. :returns: final dictionary of slot names, after merging with permanent slots and performing sanity checks. """ slots: Dict[str, Dict[str, str]] = self._get_members_slots(my_name, role) - permanent_slots: Dict[str, Any] = self._get_permanent_slots(is_standby_cluster=is_standby_cluster, - role=role, nofailover=nofailover, + permanent_slots: Dict[str, Any] = self._get_permanent_slots(role=role, nofailover=nofailover, major_version=major_version) disabled_permanent_logical_slots: List[str] = self._merge_permanent_slots( @@ -1058,8 +1033,7 @@ class Cluster(NamedTuple('Cluster', logger.error("Bad value for slot '%s' in permanent_slots: %s", name, permanent_slots[name]) return disabled_permanent_logical_slots - def _get_permanent_slots(self, *, is_standby_cluster: bool, role: str, - nofailover: bool, major_version: int) -> Dict[str, Any]: + def _get_permanent_slots(self, *, role: str, nofailover: bool, major_version: int) -> Dict[str, Any]: """Get configured permanent replication slots. .. note:: @@ -1071,18 +1045,16 @@ class Cluster(NamedTuple('Cluster', The returned dictionary for a non-standby cluster always contains permanent logical replication slots in order to show a warning if they are not supported by PostgreSQL before v11. - :param is_standby_cluster: ``True`` if it is known that this is a standby cluster. We pass the value from - the outside because we want to protect from the ``/config`` key removal. :param role: role of this node -- ``primary``, ``standby_leader`` or ``replica``. :param nofailover: ``True`` if this node is tagged to not be a failover candidate. :param major_version: postgresql major version. :returns: dictionary of permanent slot names mapped to attributes. """ - if not self.use_slots or nofailover: + if not global_config.use_slots or nofailover: return {} - if is_standby_cluster: + if global_config.is_standby_cluster: return self.__permanent_physical_slots \ if major_version >= SLOT_ADVANCE_AVAILABLE_VERSION or role == 'standby_leader' else {} @@ -1108,7 +1080,7 @@ class Cluster(NamedTuple('Cluster', :returns: dictionary of physical replication slots that should exist on a given node. """ - if not self.use_slots: + if not global_config.use_slots: return {} # we always want to exclude the member with our name from the list @@ -1132,13 +1104,11 @@ class Cluster(NamedTuple('Cluster', for k, v in slot_conflicts.items() if len(v) > 1)) return slots - def has_permanent_slots(self, my_name: str, *, is_standby_cluster: bool = False, nofailover: bool = False, + def has_permanent_slots(self, my_name: str, *, nofailover: bool = False, major_version: int = SLOT_ADVANCE_AVAILABLE_VERSION) -> bool: """Check if the given member node has permanent replication slots configured. :param my_name: name of the member node to check. - :param is_standby_cluster: ``True`` if it is known that this is a standby cluster. We pass the value from - the outside because we want to protect from the ``/config`` key removal. :param nofailover: ``True`` if this node is tagged to not be a failover candidate. :param major_version: postgresql major version. @@ -1146,20 +1116,16 @@ class Cluster(NamedTuple('Cluster', """ role = 'replica' members_slots: Dict[str, Dict[str, str]] = self._get_members_slots(my_name, role) - permanent_slots: Dict[str, Any] = self._get_permanent_slots(is_standby_cluster=is_standby_cluster, - role=role, nofailover=nofailover, + permanent_slots: Dict[str, Any] = self._get_permanent_slots(role=role, nofailover=nofailover, major_version=major_version) slots = deepcopy(members_slots) self._merge_permanent_slots(slots, permanent_slots, my_name, major_version) return len(slots) > len(members_slots) or any(self.is_physical_slot(v) for v in permanent_slots.values()) - def filter_permanent_slots(self, slots: Dict[str, int], is_standby_cluster: bool, - major_version: int) -> Dict[str, int]: + def filter_permanent_slots(self, slots: Dict[str, int], major_version: int) -> Dict[str, int]: """Filter out all non-permanent slots from provided *slots* dict. :param slots: slot names with LSN values - :param is_standby_cluster: ``True`` if it is known that this is a standby cluster. We pass the value from - the outside because we want to protect from the ``/config`` key removal. :param major_version: postgresql major version. :returns: a :class:`dict` object that contains only slots that are known to be permanent. @@ -1167,9 +1133,7 @@ class Cluster(NamedTuple('Cluster', if major_version < SLOT_ADVANCE_AVAILABLE_VERSION: return {} # for legacy PostgreSQL we don't support permanent slots on standby nodes - permanent_slots: Dict[str, Any] = self._get_permanent_slots(is_standby_cluster=is_standby_cluster, - role='replica', - nofailover=False, + permanent_slots: Dict[str, Any] = self._get_permanent_slots(role='replica', nofailover=False, major_version=major_version) members_slots = {slot_name_from_member_name(m.name) for m in self.members} @@ -1203,7 +1167,7 @@ class Cluster(NamedTuple('Cluster', if self._has_permanent_logical_slots(my_name, nofailover): return True - if self.use_slots: + if global_config.use_slots: members = [m for m in self.members if m.replicatefrom == my_name and m.name != self.leader_name] return any(self.should_enforce_hot_standby_feedback(m.name, m.nofailover) for m in members) return False diff --git a/patroni/global_config.py b/patroni/global_config.py new file mode 100644 index 00000000..7731cf59 --- /dev/null +++ b/patroni/global_config.py @@ -0,0 +1,227 @@ +"""Implements *global_config* facilities. + +The :class:`GlobalConfig` object is instantiated on import and replaces +``patroni.global_config`` module in :data:`sys.modules`, what allows to use +its properties and methods like they were module variables and functions. +""" +import sys +import types + +from copy import deepcopy +from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING + +from .utils import parse_bool, parse_int + +if TYPE_CHECKING: # pragma: no cover + from .dcs import Cluster + + +def __getattr__(mod: types.ModuleType, name: str) -> Any: + """This function exists just to make pyright happy. + + Without it pyright complains about access to unknown members of global_config module. + """ + return getattr(sys.modules[__name__], name) # pragma: no cover + + +class GlobalConfig(types.ModuleType): + """A class that wraps global configuration and provides convenient methods to access/check values.""" + + __file__ = __file__ # just to make unittest and pytest happy + + def __init__(self) -> None: + """Initialize :class:`GlobalConfig` object.""" + super().__init__(__name__) + self.__config = {} + + @staticmethod + def _cluster_has_valid_config(cluster: Optional['Cluster']) -> bool: + """Check if provided *cluster* object has a valid global configuration. + + :param cluster: the currently known cluster state from DCS. + + :returns: ``True`` if provided *cluster* object has a valid global configuration, otherwise ``False``. + """ + return bool(cluster and cluster.config and cluster.config.modify_version) + + def update(self, cluster: Optional['Cluster']) -> None: + """Update with the new global configuration from the :class:`Cluster` object view. + + .. note:: + Global configuration is updated only when configuration in the *cluster* view is valid. + + Update happens in-place and is executed only from the main heartbeat thread. + + :param cluster: the currently known cluster state from DCS. + """ + # Try to protect from the case when DCS was wiped out + if self._cluster_has_valid_config(cluster): + self.__config = cluster.config.data # pyright: ignore [reportOptionalMemberAccess] + + def from_cluster(self, cluster: Optional['Cluster']) -> 'GlobalConfig': + """Return :class:`GlobalConfig` instance from the provided :class:`Cluster` object view. + + .. note:: + If the provided *cluster* object doesn't have a valid global configuration we return + the last known valid state of the :class:`GlobalConfig` object. + + This method is used when we need to have the most up-to-date values in the global configuration, + but we don't want to update the global object. + + :param cluster: the currently known cluster state from DCS. + + :returns: :class:`GlobalConfig` object. + """ + if not self._cluster_has_valid_config(cluster): + return self + + ret = GlobalConfig() + ret.update(cluster) + return ret + + def get(self, name: str) -> Any: + """Gets global configuration value by *name*. + + :param name: parameter name. + + :returns: configuration value or ``None`` if it is missing. + """ + return self.__config.get(name) + + def check_mode(self, mode: str) -> bool: + """Checks whether the certain parameter is enabled. + + :param mode: parameter name, e.g. ``synchronous_mode``, ``failsafe_mode``, ``pause``, ``check_timeline``, and + so on. + + :returns: ``True`` if parameter *mode* is enabled in the global configuration. + """ + return bool(parse_bool(self.__config.get(mode))) + + @property + def is_paused(self) -> bool: + """``True`` if cluster is in maintenance mode.""" + return self.check_mode('pause') + + @property + def is_synchronous_mode(self) -> bool: + """``True`` if synchronous replication is requested and it is not a standby cluster config.""" + return self.check_mode('synchronous_mode') and not self.is_standby_cluster + + @property + def is_synchronous_mode_strict(self) -> bool: + """``True`` if at least one synchronous node is required.""" + return self.check_mode('synchronous_mode_strict') + + def get_standby_cluster_config(self) -> Union[Dict[str, Any], Any]: + """Get ``standby_cluster`` configuration. + + :returns: a copy of ``standby_cluster`` configuration. + """ + return deepcopy(self.get('standby_cluster')) + + @property + def is_standby_cluster(self) -> bool: + """``True`` if global configuration has a valid ``standby_cluster`` section.""" + config = self.get_standby_cluster_config() + return isinstance(config, dict) and\ + bool(config.get('host') or config.get('port') or config.get('restore_command')) + + def get_int(self, name: str, default: int = 0) -> int: + """Gets current value of *name* from the global configuration and try to return it as :class:`int`. + + :param name: name of the parameter. + :param default: default value if *name* is not in the configuration or invalid. + + :returns: currently configured value of *name* from the global configuration or *default* if it is not set or + invalid. + """ + ret = parse_int(self.get(name)) + return default if ret is None else ret + + @property + def min_synchronous_nodes(self) -> int: + """The minimum number of synchronous nodes based on whether ``synchronous_mode_strict`` is enabled or not.""" + return 1 if self.is_synchronous_mode_strict else 0 + + @property + def synchronous_node_count(self) -> int: + """Currently configured value of ``synchronous_node_count`` from the global configuration. + + Assume ``1`` if it is not set or invalid. + """ + return max(self.get_int('synchronous_node_count', 1), self.min_synchronous_nodes) + + @property + def maximum_lag_on_failover(self) -> int: + """Currently configured value of ``maximum_lag_on_failover`` from the global configuration. + + Assume ``1048576`` if it is not set or invalid. + """ + return self.get_int('maximum_lag_on_failover', 1048576) + + @property + def maximum_lag_on_syncnode(self) -> int: + """Currently configured value of ``maximum_lag_on_syncnode`` from the global configuration. + + Assume ``-1`` if it is not set or invalid. + """ + return self.get_int('maximum_lag_on_syncnode', -1) + + @property + def primary_start_timeout(self) -> int: + """Currently configured value of ``primary_start_timeout`` from the global configuration. + + Assume ``300`` if it is not set or invalid. + + .. note:: + ``master_start_timeout`` is still supported to keep backward compatibility. + """ + default = 300 + return self.get_int('primary_start_timeout', default)\ + if 'primary_start_timeout' in self.__config else self.get_int('master_start_timeout', default) + + @property + def primary_stop_timeout(self) -> int: + """Currently configured value of ``primary_stop_timeout`` from the global configuration. + + Assume ``0`` if it is not set or invalid. + + .. note:: + ``master_stop_timeout`` is still supported to keep backward compatibility. + """ + default = 0 + return self.get_int('primary_stop_timeout', default)\ + if 'primary_stop_timeout' in self.__config else self.get_int('master_stop_timeout', default) + + @property + def ignore_slots_matchers(self) -> List[Dict[str, Any]]: + """Currently configured value of ``ignore_slots`` from the global configuration. + + Assume an empty :class:`list` if not set. + """ + return self.get('ignore_slots') or [] + + @property + def max_timelines_history(self) -> int: + """Currently configured value of ``max_timelines_history`` from the global configuration. + + Assume ``0`` if not set or invalid. + """ + return self.get_int('max_timelines_history', 0) + + @property + def use_slots(self) -> bool: + """``True`` if cluster is configured to use replication slots.""" + return bool(parse_bool((self.get('postgresql') or {}).get('use_slots', True))) + + @property + def permanent_slots(self) -> Dict[str, Any]: + """Dictionary of permanent slots information from the global configuration.""" + return deepcopy(self.get('permanent_replication_slots') + or self.get('permanent_slots') + or self.get('slots') + or {}) + + +sys.modules[__name__] = GlobalConfig() diff --git a/patroni/ha.py b/patroni/ha.py index 4a21972a..12927526 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -10,7 +10,7 @@ from multiprocessing.pool import ThreadPool from threading import RLock from typing import Any, Callable, Collection, Dict, List, NamedTuple, Optional, Union, Tuple, TYPE_CHECKING -from . import psycopg +from . import global_config, psycopg from .__main__ import Patroni from .async_executor import AsyncExecutor, CriticalTask from .collections import CaseInsensitiveSet @@ -156,7 +156,6 @@ class Ha(object): self._rewind = Rewind(self.state_handler) self.dcs = patroni.dcs self.cluster = Cluster.empty() - self.global_config = self.patroni.config.get_global_config(None) self.old_cluster = Cluster.empty() self._leader_expiry = 0 self._leader_expiry_lock = RLock() @@ -188,20 +187,20 @@ class Ha(object): def primary_stop_timeout(self) -> Union[int, None]: """:returns: "primary_stop_timeout" from the global configuration or `None` when not in synchronous mode.""" - ret = self.global_config.primary_stop_timeout + ret = global_config.primary_stop_timeout return ret if ret > 0 and self.is_synchronous_mode() else None def is_paused(self) -> bool: """:returns: `True` if in maintenance mode.""" - return self.global_config.is_paused + return global_config.is_paused def check_timeline(self) -> bool: """:returns: `True` if should check whether the timeline is latest during the leader race.""" - return self.global_config.check_mode('check_timeline') + return global_config.check_mode('check_timeline') def is_standby_cluster(self) -> bool: """:returns: `True` if global configuration has a valid "standby_cluster" section.""" - return self.global_config.is_standby_cluster + return global_config.is_standby_cluster def is_leader(self) -> bool: """:returns: `True` if the current node is the leader, based on expiration set when it last held the key.""" @@ -296,7 +295,6 @@ class Ha(object): last_lsn = self.state_handler.last_operation() slots = self.cluster.filter_permanent_slots( {**self.state_handler.slots(), slot_name_from_member_name(self.state_handler.name): last_lsn}, - self.is_standby_cluster(), self.state_handler.major_version) except Exception: logger.exception('Exception when called state_handler.last_operation()') @@ -450,7 +448,7 @@ class Ha(object): return ret or 'trying to bootstrap {0}'.format(msg) # no leader, but configuration may allowed replica creation using backup tools - create_replica_methods = self.global_config.get_standby_cluster_config().get('create_replica_methods', []) \ + create_replica_methods = global_config.get_standby_cluster_config().get('create_replica_methods', []) \ if self.is_standby_cluster() else None can_bootstrap = self.state_handler.can_create_replica_without_replication_connection(create_replica_methods) concurrent_bootstrap = self.cluster.initialize == "" @@ -525,7 +523,7 @@ class Ha(object): :returns: action message, describing what was performed. """ if self.has_lock() and self.update_lock(): - timeout = self.global_config.primary_start_timeout + timeout = global_config.primary_start_timeout if timeout == 0: # We are requested to prefer failing over to restarting primary. But see first if there # is anyone to fail over to. @@ -622,7 +620,7 @@ class Ha(object): for param in params: # It is highly unlikely to happen, but we want to protect from the case node_to_follow.data.pop(param, None) # when above-mentioned params came from outside. if self.is_standby_cluster(): - standby_config = self.global_config.get_standby_cluster_config() + standby_config = global_config.get_standby_cluster_config() node_to_follow.data.update({p: standby_config[p] for p in params if standby_config.get(p)}) return node_to_follow @@ -684,11 +682,11 @@ class Ha(object): def is_synchronous_mode(self) -> bool: """:returns: `True` if synchronous replication is requested.""" - return self.global_config.is_synchronous_mode + return global_config.is_synchronous_mode def is_failsafe_mode(self) -> bool: """:returns: `True` if failsafe_mode is enabled in global configuration.""" - return self.global_config.check_mode('failsafe_mode') + return global_config.check_mode('failsafe_mode') def process_sync_replication(self) -> None: """Process synchronous standby beahvior. @@ -732,7 +730,7 @@ class Ha(object): return logger.info('Synchronous replication key updated by someone else.') # When strict mode and no suitable replication connections put "*" to synchronous_standby_names - if self.global_config.is_synchronous_mode_strict and not picked: + if global_config.is_synchronous_mode_strict and not picked: picked = CaseInsensitiveSet('*') logger.warning("No standbys available!") @@ -805,7 +803,7 @@ class Ha(object): cluster_history_dict: Dict[int, List[Any]] = {line[0]: list(line) for line in cluster_history} history: List[List[Any]] = list(map(list, self.state_handler.get_history(primary_timeline))) if self.cluster.config: - history = history[-self.cluster.config.max_timelines_history:] + history = history[-global_config.max_timelines_history:] for line in history: # enrich current history with promotion timestamps stored in DCS cluster_history_line = cluster_history_dict.get(line[0], []) @@ -863,7 +861,7 @@ class Ha(object): # promotion until next cycle. TODO: trigger immediate retry of run_cycle return 'Postponing promotion because synchronous replication state was updated by somebody else' self.state_handler.sync_handler.set_synchronous_standby_names( - CaseInsensitiveSet('*') if self.global_config.is_synchronous_mode_strict else CaseInsensitiveSet()) + CaseInsensitiveSet('*') if global_config.is_synchronous_mode_strict else CaseInsensitiveSet()) if self.state_handler.role not in ('master', 'promoted', 'primary'): # reset failsafe state when promote self._failsafe.set_is_active(0) @@ -974,7 +972,7 @@ class Ha(object): :returns True when node is lagging """ lag = (self.cluster.last_lsn or 0) - wal_position - return lag > self.global_config.maximum_lag_on_failover + return lag > global_config.maximum_lag_on_failover def _is_healthiest_node(self, members: Collection[Member], check_replication_lag: bool = True) -> bool: """This method tries to determine whether I am healthy enough to became a new leader candidate or not.""" @@ -1541,7 +1539,7 @@ class Ha(object): # Now that restart is scheduled we can set timeout for startup, it will get reset # once async executor runs and main loop notices PostgreSQL as up. - timeout = restart_data.get('timeout', self.global_config.primary_start_timeout) + timeout = restart_data.get('timeout', global_config.primary_start_timeout) self.set_start_timeout(timeout) def before_shutdown() -> None: @@ -1605,7 +1603,7 @@ class Ha(object): """Figure out what to do with the task AsyncExecutor is performing.""" if self.has_lock() and self.update_lock(): if self._async_executor.scheduled_action == 'doing crash recovery in a single user mode': - time_left = self.global_config.primary_start_timeout - (time.time() - self._crash_recovery_started) + time_left = global_config.primary_start_timeout - (time.time() - self._crash_recovery_started) if time_left <= 0 and self.is_failover_possible(): logger.info("Demoting self because crash recovery is taking too long") self.state_handler.cancellable.cancel(True) @@ -1690,7 +1688,7 @@ class Ha(object): self.set_is_leader(True) if self.is_synchronous_mode(): self.state_handler.sync_handler.set_synchronous_standby_names( - CaseInsensitiveSet('*') if self.global_config.is_synchronous_mode_strict else CaseInsensitiveSet()) + CaseInsensitiveSet('*') if global_config.is_synchronous_mode_strict else CaseInsensitiveSet()) self.state_handler.call_nowait(CallbackAction.ON_START) self.load_cluster_from_dcs() @@ -1713,7 +1711,7 @@ class Ha(object): self.demote('immediate-nolock') return 'stopped PostgreSQL while starting up because leader key was lost' - timeout = self._start_timeout or self.global_config.primary_start_timeout + timeout = self._start_timeout or global_config.primary_start_timeout time_left = timeout - self.state_handler.time_in_state() if time_left <= 0: @@ -1746,8 +1744,8 @@ class Ha(object): try: try: self.load_cluster_from_dcs() - self.global_config = self.patroni.config.get_global_config(self.cluster) - self.state_handler.reset_cluster_info_state(self.cluster, self.patroni.nofailover, self.global_config) + global_config.update(self.cluster) + self.state_handler.reset_cluster_info_state(self.cluster, self.patroni.nofailover) except Exception: self.state_handler.reset_cluster_info_state(None) raise @@ -1767,10 +1765,10 @@ class Ha(object): self.touch_member() # cluster has leader key but not initialize key - if not (self.cluster.is_unlocked() or self.sysid_valid(self.cluster.initialize)) and self.has_lock(): + if self.has_lock(False) and not self.sysid_valid(self.cluster.initialize): self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=self.state_handler.sysid) - if not (self.cluster.is_unlocked() or self.cluster.config and self.cluster.config.data) and self.has_lock(): + if self.has_lock(False) and not (self.cluster.config and self.cluster.config.data): self.dcs.set_config_value(json.dumps(self.patroni.config.dynamic_configuration, separators=(',', ':'))) self.cluster = self.dcs.get_cluster() @@ -2047,7 +2045,7 @@ class Ha(object): config or cluster.config.data. """ data: Dict[str, Any] = {} - cluster_params = self.global_config.get_standby_cluster_config() + cluster_params = global_config.get_standby_cluster_config() if cluster_params: data.update({k: v for k, v in cluster_params.items() if k in RemoteMember.ALLOWED_KEYS}) diff --git a/patroni/postgresql/__init__.py b/patroni/postgresql/__init__.py index fdfc26c1..c1027dad 100644 --- a/patroni/postgresql/__init__.py +++ b/patroni/postgresql/__init__.py @@ -24,7 +24,7 @@ from .misc import parse_history, parse_lsn, postgres_major_version_to_int from .postmaster import PostmasterProcess from .slots import SlotsHandler from .sync import SyncHandler -from .. import psycopg +from .. import global_config, psycopg from ..async_executor import CriticalTask from ..collections import CaseInsensitiveSet from ..dcs import Cluster, Leader, Member, SLOT_ADVANCE_AVAILABLE_VERSION @@ -34,7 +34,6 @@ from ..utils import Retry, RetryFailedError, polling_loop, data_directory_is_emp if TYPE_CHECKING: # pragma: no cover from psycopg import Connection as Connection3, Cursor from psycopg2 import connection as connection3, cursor - from ..config import GlobalConfig logger = logging.getLogger(__name__) @@ -73,7 +72,6 @@ class Postgresql(object): self.connection_string: str self.proxy_url: Optional[str] self._major_version = self.get_major_version() - self._global_config = None self._state_lock = Lock() self.set_state('stopped') @@ -217,7 +215,7 @@ class Postgresql(object): "FROM pg_catalog.pg_stat_get_wal_senders() w," " pg_catalog.pg_stat_get_activity(w.pid)" " WHERE w.state = 'streaming') r)").format(self.wal_name, self.lsn_name) - if (not self.global_config or self.global_config.is_synchronous_mode) + if global_config.is_synchronous_mode and self.role in ('master', 'primary', 'promoted') else "'on', '', NULL") if self._major_version >= 90600: @@ -426,12 +424,7 @@ class Postgresql(object): self.config.write_postgresql_conf() self.reload() - @property - def global_config(self) -> Optional['GlobalConfig']: - return self._global_config - - def reset_cluster_info_state(self, cluster: Union[Cluster, None], nofailover: bool = False, - global_config: Optional['GlobalConfig'] = None) -> None: + def reset_cluster_info_state(self, cluster: Union[Cluster, None], nofailover: bool = False) -> None: """Reset monitoring query cache. It happens in the beginning of heart-beat loop and on change of `synchronous_standby_names`. @@ -440,30 +433,22 @@ class Postgresql(object): :param nofailover: whether this node could become a new primary. Important when there are logical permanent replication slots because "nofailover" node could do cascading replication and should enable `hot_standby_feedback` - :param global_config: last known :class:`GlobalConfig` object """ self._cluster_info_state = {} - if global_config: - self._global_config = global_config - - if not self._global_config: - return - - if self._global_config.is_standby_cluster: + if global_config.is_standby_cluster: # Standby cluster can't have logical replication slots, and we don't need to enforce hot_standby_feedback self.set_enforce_hot_standby_feedback(False) if cluster and cluster.config and cluster.config.modify_version: # We want to enable hot_standby_feedback if the replica is supposed # to have a logical slot or in case if it is the cascading replica. - self.set_enforce_hot_standby_feedback(not self._global_config.is_standby_cluster and self.can_advance_slots + self.set_enforce_hot_standby_feedback(not global_config.is_standby_cluster and self.can_advance_slots and cluster.should_enforce_hot_standby_feedback(self.name, nofailover)) self._has_permanent_slots = cluster.has_permanent_slots( my_name=self.name, - is_standby_cluster=self._global_config.is_standby_cluster, nofailover=nofailover, major_version=self.major_version) diff --git a/patroni/postgresql/config.py b/patroni/postgresql/config.py index b15f7f23..271bbdfe 100644 --- a/patroni/postgresql/config.py +++ b/patroni/postgresql/config.py @@ -12,6 +12,7 @@ from types import TracebackType from typing import Any, Collection, Dict, Iterator, List, Optional, Union, Tuple, Type, TYPE_CHECKING from .validator import recovery_parameters, transform_postgresql_parameter_value, transform_recovery_parameter_value +from .. import global_config from ..collections import CaseInsensitiveDict, CaseInsensitiveSet from ..dcs import Leader, Member, RemoteMember, slot_name_from_member_name from ..exceptions import PatroniFatalException, PostgresConnectionException @@ -595,7 +596,7 @@ class ConfigHandler(object): is_remote_member = isinstance(member, RemoteMember) primary_conninfo = self.primary_conninfo_params(member) if primary_conninfo: - use_slots = self.get('use_slots', True) and self._postgresql.major_version >= 90400 + use_slots = global_config.use_slots and self._postgresql.major_version >= 90400 if use_slots and not (is_remote_member and member.no_replication_slot): primary_slot_name = member.primary_slot_name if is_remote_member else self._postgresql.name recovery_params['primary_slot_name'] = slot_name_from_member_name(primary_slot_name) @@ -930,10 +931,10 @@ class ConfigHandler(object): parameters = config['parameters'].copy() listen_addresses, port = split_host_port(config['listen'], 5432) parameters.update(cluster_name=self._postgresql.scope, listen_addresses=listen_addresses, port=str(port)) - if not self._postgresql.global_config or self._postgresql.global_config.is_synchronous_mode: + if global_config.is_synchronous_mode: synchronous_standby_names = self._server_parameters.get('synchronous_standby_names') if synchronous_standby_names is None: - if self._postgresql.global_config and self._postgresql.global_config.is_synchronous_mode_strict\ + if global_config.is_synchronous_mode_strict\ and self._postgresql.role in ('master', 'primary', 'promoted'): parameters['synchronous_standby_names'] = '*' else: diff --git a/patroni/postgresql/slots.py b/patroni/postgresql/slots.py index 48b275e4..66bb1f00 100644 --- a/patroni/postgresql/slots.py +++ b/patroni/postgresql/slots.py @@ -13,6 +13,7 @@ from typing import Any, Dict, Iterator, List, Optional, Union, Tuple, TYPE_CHECK from .connection import get_connection_cursor from .misc import format_lsn, fsync_dir +from .. import global_config from ..dcs import Cluster, Leader from ..file_perm import pg_perm from ..psycopg import OperationalError @@ -293,7 +294,7 @@ class SlotsHandler: """ slot = self._replication_slots[name] if cluster.config: - for matcher in cluster.config.ignore_slots_matchers: + for matcher in global_config.ignore_slots_matchers: if ( (matcher.get("name") is None or matcher["name"] == name) and all(not matcher.get(a) or matcher[a] == slot.get(a) @@ -510,13 +511,12 @@ class SlotsHandler: :returns: list of logical replication slots names that should be copied from the primary. """ ret = [] - if self._postgresql.major_version >= 90400 and self._postgresql.global_config and cluster.config: + if self._postgresql.major_version >= 90400 and cluster.config: try: self.load_replication_slots() - slots = cluster.get_replication_slots( - self._postgresql.name, self._postgresql.role, nofailover, self._postgresql.major_version, - is_standby_cluster=self._postgresql.global_config.is_standby_cluster, show_error=True) + slots = cluster.get_replication_slots(self._postgresql.name, self._postgresql.role, + nofailover, self._postgresql.major_version, show_error=True) self._drop_incorrect_slots(cluster, slots, paused) diff --git a/patroni/postgresql/sync.py b/patroni/postgresql/sync.py index 9cff04e0..577422b5 100644 --- a/patroni/postgresql/sync.py +++ b/patroni/postgresql/sync.py @@ -5,6 +5,7 @@ import time from copy import deepcopy from typing import Collection, List, NamedTuple, Tuple, TYPE_CHECKING +from .. import global_config from ..collections import CaseInsensitiveDict, CaseInsensitiveSet from ..dcs import Cluster from ..psycopg import quote_ident as _quote_ident @@ -303,11 +304,8 @@ END;$$""") replica_list = _ReplicaList(self._postgresql, cluster) self._process_replica_readiness(cluster, replica_list) - if TYPE_CHECKING: # pragma: no cover - assert self._postgresql.global_config is not None - sync_node_count = self._postgresql.global_config.synchronous_node_count\ - if self._postgresql.supports_multiple_sync else 1 - sync_node_maxlag = self._postgresql.global_config.maximum_lag_on_syncnode + sync_node_count = global_config.synchronous_node_count if self._postgresql.supports_multiple_sync else 1 + sync_node_maxlag = global_config.maximum_lag_on_syncnode candidates = CaseInsensitiveSet() sync_nodes = CaseInsensitiveSet() diff --git a/patroni/utils.py b/patroni/utils.py index 6957369f..23f419e5 100644 --- a/patroni/utils.py +++ b/patroni/utils.py @@ -33,7 +33,6 @@ from .version import __version__ if TYPE_CHECKING: # pragma: no cover from .dcs import Cluster - from .config import GlobalConfig tzutc = tz.tzutc() @@ -759,12 +758,10 @@ def iter_response_objects(response: HTTPResponse) -> Iterator[Dict[str, Any]]: prev = chunk[idx:] -def cluster_as_json(cluster: 'Cluster', global_config: Optional['GlobalConfig'] = None) -> Dict[str, Any]: +def cluster_as_json(cluster: 'Cluster') -> Dict[str, Any]: """Get a JSON representation of *cluster*. :param cluster: the :class:`~patroni.dcs.Cluster` object to be parsed as JSON. - :param global_config: optional :class:`~patroni.config.GlobalConfig` object to check the cluster state. - if not provided will be instantiated from the `Cluster.config`. :returns: JSON representation of *cluster*. @@ -793,16 +790,16 @@ def cluster_as_json(cluster: 'Cluster', global_config: Optional['GlobalConfig'] * ``from``: name of the member to be demoted; * ``to``: name of the member to be promoted. """ - if not global_config: - from patroni.config import get_global_config - global_config = get_global_config(cluster) + from . import global_config + + config = global_config.from_cluster(cluster) leader_name = cluster.leader.name if cluster.leader else None cluster_lsn = cluster.last_lsn or 0 ret: Dict[str, Any] = {'members': []} for m in cluster.members: if m.name == leader_name: - role = 'standby_leader' if global_config.is_standby_cluster else 'leader' + role = 'standby_leader' if config.is_standby_cluster else 'leader' elif cluster.sync.matches(m.name): role = 'sync_standby' else: @@ -832,7 +829,7 @@ def cluster_as_json(cluster: 'Cluster', global_config: Optional['GlobalConfig'] # sort members by name for consistency cmp: Callable[[Dict[str, Any]], bool] = lambda m: m['name'] ret['members'].sort(key=cmp) - if global_config.is_paused: + if config.is_paused: ret['pause'] = True if cluster.failover and cluster.failover.scheduled_at: ret['scheduled_switchover'] = {'at': cluster.failover.scheduled_at.isoformat()} diff --git a/tests/test_api.py b/tests/test_api.py index 71c566cd..234a6824 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -8,8 +8,8 @@ from io import BytesIO as IO from mock import Mock, PropertyMock, patch from socketserver import ThreadingMixIn +from patroni import global_config from patroni.api import RestApiHandler, RestApiServer -from patroni.config import GlobalConfig from patroni.dcs import ClusterConfig, Member from patroni.exceptions import PostgresConnectionException from patroni.ha import _MemberStatus @@ -148,16 +148,9 @@ class MockLogger(object): records_lost = 1 -class MockConfig(object): - - def get_global_config(self, _): - return GlobalConfig({}) - - class MockPatroni(object): ha = MockHa() - config = MockConfig() postgresql = ha.state_handler dcs = Mock() logger = MockLogger() @@ -211,7 +204,7 @@ class TestRestApiHandler(unittest.TestCase): def test_do_GET(self): MockPatroni.dcs.cluster.last_lsn = 20 MockPatroni.dcs.cluster.sync.members = [MockPostgresql.name] - with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=True)): + with patch.object(global_config.__class__, 'is_synchronous_mode', PropertyMock(return_value=True)): MockRestApiServer(RestApiHandler, 'GET /replica') MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M') MockRestApiServer(RestApiHandler, 'GET /replica?lag=10MB') @@ -234,7 +227,7 @@ class TestRestApiHandler(unittest.TestCase): with patch.object(MockHa, 'is_leader', Mock(return_value=True)): MockRestApiServer(RestApiHandler, 'GET /replica') MockRestApiServer(RestApiHandler, 'GET /read-only-sync') - with patch.object(GlobalConfig, 'is_standby_cluster', Mock(return_value=True)): + with patch.object(global_config.__class__, 'is_standby_cluster', Mock(return_value=True)): MockRestApiServer(RestApiHandler, 'GET /standby_leader') MockPatroni.dcs.cluster = None with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'primary'})): @@ -244,8 +237,8 @@ class TestRestApiHandler(unittest.TestCase): self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /primary')) with patch.object(RestApiServer, 'query', Mock(return_value=[('', 1, '', '', '', '', False, None, None, '')])): self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni')) - with patch.object(GlobalConfig, 'is_standby_cluster', Mock(return_value=True)), \ - patch.object(GlobalConfig, 'is_paused', Mock(return_value=True)): + with patch.object(global_config.__class__, 'is_standby_cluster', Mock(return_value=True)), \ + patch.object(global_config.__class__, 'is_paused', Mock(return_value=True)): MockRestApiServer(RestApiHandler, 'GET /standby_leader') # test tags @@ -475,7 +468,7 @@ class TestRestApiHandler(unittest.TestCase): request = make_request(role='primary', postgres_version='9.5.2') MockRestApiServer(RestApiHandler, request) - with patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)): + with patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)): MockRestApiServer(RestApiHandler, make_request(schedule='2016-08-42 12:45TZ+1', role='primary')) # Valid timeout MockRestApiServer(RestApiHandler, make_request(timeout='60s')) @@ -537,7 +530,7 @@ class TestRestApiHandler(unittest.TestCase): # Switchover in pause mode with patch.object(RestApiHandler, 'write_response') as response_mock, \ - patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)): + patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)): MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with( 400, 'Switchover is possible only to a specific candidate in a paused state') @@ -546,7 +539,8 @@ class TestRestApiHandler(unittest.TestCase): for is_synchronous_mode, response in ( (True, 'switchover is not possible: can not find sync_standby'), (False, 'switchover is not possible: cluster does not have members except leader')): - with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \ + with patch.object(global_config.__class__, 'is_synchronous_mode', + PropertyMock(return_value=is_synchronous_mode)), \ patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with(412, response) @@ -571,7 +565,8 @@ class TestRestApiHandler(unittest.TestCase): cluster.sync.matches.return_value = False for is_synchronous_mode, response in ( (True, 'candidate name does not match with sync_standby'), (False, 'candidate does not exists')): - with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \ + with patch.object(global_config.__class__, 'is_synchronous_mode', + PropertyMock(return_value=is_synchronous_mode)), \ patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with(412, response) @@ -632,7 +627,7 @@ class TestRestApiHandler(unittest.TestCase): # Schedule in paused mode with patch.object(RestApiHandler, 'write_response') as response_mock, \ - patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)): + patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)): dcs.manual_failover.return_value = False MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with(400, "Can't schedule switchover in the paused state") diff --git a/tests/test_config.py b/tests/test_config.py index dfb3b6e3..7bf01f56 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,7 +5,11 @@ import io from copy import deepcopy from mock import MagicMock, Mock, patch -from patroni.config import Config, ConfigParseError, GlobalConfig + +from patroni import global_config +from patroni.config import ClusterConfig, Config, ConfigParseError + +from .test_ha import get_cluster_initialized_with_only_leader class TestConfig(unittest.TestCase): @@ -248,4 +252,6 @@ class TestConfig(unittest.TestCase): def test_global_config_is_synchronous_mode(self): # we should ignore synchronous_mode setting in a standby cluster config = {'standby_cluster': {'host': 'some_host'}, 'synchronous_mode': True} - self.assertFalse(GlobalConfig(config).is_synchronous_mode) + cluster = get_cluster_initialized_with_only_leader(cluster_config=ClusterConfig(1, config, 1)) + test_config = global_config.from_cluster(cluster) + self.assertFalse(test_config.is_synchronous_mode) diff --git a/tests/test_ctl.py b/tests/test_ctl.py index 96c36c16..a174b03d 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -7,6 +7,7 @@ import unittest from click.testing import CliRunner from datetime import datetime, timedelta from mock import patch, Mock, PropertyMock +from patroni import global_config from patroni.ctl import ctl, load_config, output_members, get_dcs, parse_dcs, \ get_all_members, get_any_member, get_cursor, query_member, PatroniCtlException, apply_config_changes, \ format_config_for_editing, show_diff, invoke_editor, format_pg_version, CONFIG_FILE_PATH, PatronictlPrettyTable @@ -147,7 +148,7 @@ class TestCtl(unittest.TestCase): self.assertEqual(result.exit_code, 0) # Scheduled in pause mode - with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)): + with patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)): result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force', '--scheduled', '2015-01-01T12:00:00']) self.assertEqual(result.exit_code, 1) @@ -369,7 +370,7 @@ class TestCtl(unittest.TestCase): result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30']) assert 'Failed: flush scheduled restart' in result.output - with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)): + with patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)): result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30']) assert result.exit_code == 1 @@ -533,7 +534,7 @@ class TestCtl(unittest.TestCase): result = self.runner.invoke(ctl, ['pause', 'dummy']) assert 'Failed' in result.output - with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)): + with patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)): result = self.runner.invoke(ctl, ['pause', 'dummy']) assert 'Cluster is already paused' in result.output @@ -552,11 +553,11 @@ class TestCtl(unittest.TestCase): @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) def test_resume_cluster(self, mock_post): mock_post.return_value.status = 200 - with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=False)): + with patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=False)): result = self.runner.invoke(ctl, ['resume', 'dummy']) assert 'Cluster is not paused' in result.output - with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)): + with patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)): result = self.runner.invoke(ctl, ['resume', 'dummy']) assert 'Success' in result.output diff --git a/tests/test_ha.py b/tests/test_ha.py index e197d28b..40063018 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -4,6 +4,7 @@ import os import sys from mock import Mock, MagicMock, PropertyMock, patch, mock_open +from patroni import global_config from patroni.collections import CaseInsensitiveSet from patroni.config import Config from patroni.dcs import Cluster, ClusterConfig, Failover, Leader, Member, get_dcs, Status, SyncState, TimelineHistory @@ -217,6 +218,7 @@ class TestHa(PostgresInit): self.ha = Ha(MockPatroni(self.p, self.e)) self.ha.old_cluster = self.e.get_cluster() self.ha.cluster = get_cluster_initialized_without_leader() + global_config.update(self.ha.cluster) self.ha.load_cluster_from_dcs = Mock() def test_update_lock(self): @@ -251,8 +253,10 @@ class TestHa(PostgresInit): @patch('patroni.dcs.etcd.Etcd.initialize', return_value=True) def test_bootstrap_as_standby_leader(self, initialize): self.p.data_directory_empty = true + self.ha.cluster = get_cluster_not_initialized_without_leader( + cluster_config=ClusterConfig(1, {"standby_cluster": {"port": 5432}}, 1)) + global_config.update(self.ha.cluster) self.ha.cluster = get_cluster_not_initialized_without_leader(cluster_config=ClusterConfig(0, {}, 0)) - self.ha.patroni.config._dynamic_configuration = {"standby_cluster": {"port": 5432}} self.assertEqual(self.ha.run_cycle(), 'trying to bootstrap a new standby leader') def test_bootstrap_waiting_for_standby_leader(self): @@ -318,7 +322,7 @@ class TestHa(PostgresInit): self.ha.state_handler.cancellable._process = Mock() self.ha._crash_recovery_started -= 600 self.ha.cluster.config.data.update({'maximum_lag_on_failover': 10}) - self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster) + global_config.update(self.ha.cluster) self.assertEqual(self.ha.run_cycle(), 'terminated crash recovery because of startup timeout') @patch.object(Rewind, 'ensure_clean_shutdown', Mock()) @@ -509,7 +513,7 @@ class TestHa(PostgresInit): def test_check_failsafe_topology(self): self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly')) self.ha.cluster = get_cluster_initialized_with_leader_and_failsafe() - self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster) + global_config.update(self.ha.cluster) self.ha.dcs._last_failsafe = self.ha.cluster.failsafe self.assertEqual(self.ha.run_cycle(), 'demoting self because DCS is not accessible and I was a leader') self.ha.state_handler.name = self.ha.cluster.leader.name @@ -529,7 +533,7 @@ class TestHa(PostgresInit): def test_no_dcs_connection_primary_failsafe(self): self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly')) self.ha.cluster = get_cluster_initialized_with_leader_and_failsafe() - self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster) + global_config.update(self.ha.cluster) self.ha.dcs._last_failsafe = self.ha.cluster.failsafe self.ha.state_handler.name = self.ha.cluster.leader.name self.assertEqual(self.ha.run_cycle(), @@ -546,7 +550,7 @@ class TestHa(PostgresInit): def test_no_dcs_connection_replica_failsafe(self): self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly')) self.ha.cluster = get_cluster_initialized_with_leader_and_failsafe() - self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster) + global_config.update(self.ha.cluster) self.ha.update_failsafe({'name': 'leader', 'api_url': 'http://127.0.0.1:8008/patroni', 'conn_url': 'postgres://127.0.0.1:5432/postgres', 'slots': {'foo': 1000}}) self.p.is_primary = false @@ -766,7 +770,7 @@ class TestHa(PostgresInit): with patch('patroni.ha.logger.info') as mock_info: self.ha.fetch_node_status = get_node_status(wal_position=1) self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5}) - self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster) + global_config.update(self.ha.cluster) self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock') self.assertEqual(mock_info.call_args_list[0][0], ('Member %s exceeds maximum replication lag', 'leader')) @@ -1032,7 +1036,7 @@ class TestHa(PostgresInit): def test__is_healthiest_node(self): self.p.is_primary = false self.ha.cluster = get_cluster_initialized_without_leader(sync=('postgresql1', self.p.name)) - self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster) + global_config.update(self.ha.cluster) self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members)) self.ha.fetch_node_status = get_node_status() # accessible, in_recovery self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members)) @@ -1049,7 +1053,7 @@ class TestHa(PostgresInit): with patch.object(Ha, 'is_synchronous_mode', Mock(return_value=True)): self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members)) self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5}) - self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster) + global_config.update(self.ha.cluster) with patch('patroni.postgresql.Postgresql.last_operation', return_value=1): self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members)) with patch('patroni.postgresql.Postgresql.replica_cached_timeline', return_value=None): @@ -1272,7 +1276,7 @@ class TestHa(PostgresInit): self.p.is_running = false self.ha.cluster = get_cluster_initialized_with_leader(sync=(self.p.name, 'other')) self.ha.cluster.config.data.update({'synchronous_mode': True, 'primary_start_timeout': 0}) - self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster) + global_config.update(self.ha.cluster) self.ha.has_lock = true self.ha.update_lock = true self.ha.fetch_node_status = get_node_status() # accessible, in_recovery @@ -1282,13 +1286,13 @@ class TestHa(PostgresInit): def test_primary_stop_timeout(self): self.assertEqual(self.ha.primary_stop_timeout(), None) self.ha.cluster.config.data.update({'primary_stop_timeout': 30}) - self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster) + global_config.update(self.ha.cluster) with patch.object(Ha, 'is_synchronous_mode', Mock(return_value=True)): self.assertEqual(self.ha.primary_stop_timeout(), 30) with patch.object(Ha, 'is_synchronous_mode', Mock(return_value=False)): self.assertEqual(self.ha.primary_stop_timeout(), None) self.ha.cluster.config.data['primary_stop_timeout'] = None - self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster) + global_config.update(self.ha.cluster) self.assertEqual(self.ha.primary_stop_timeout(), None) @patch('patroni.postgresql.Postgresql.follow') @@ -1380,8 +1384,9 @@ class TestHa(PostgresInit): # Test sync set to '*' when synchronous_mode_strict is enabled mock_set_sync.reset_mock() self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(), CaseInsensitiveSet())) - with patch('patroni.config.GlobalConfig.is_synchronous_mode_strict', PropertyMock(return_value=True)): - self.ha.run_cycle() + self.ha.cluster.config.data['synchronous_mode_strict'] = True + global_config.update(self.ha.cluster) + self.ha.run_cycle() mock_set_sync.assert_called_once_with(CaseInsensitiveSet('*')) def test_sync_replication_become_primary(self): @@ -1514,7 +1519,6 @@ class TestHa(PostgresInit): @patch('patroni.postgresql.mtime', Mock(return_value=1588316884)) @patch('builtins.open', Mock(side_effect=Exception)) - @patch.object(Cluster, 'is_unlocked', Mock(return_value=False)) def test_restore_cluster_config(self): self.ha.cluster.config.data.clear() self.ha.has_lock = true diff --git a/tests/test_patroni.py b/tests/test_patroni.py index bf9e2871..8e08d406 100644 --- a/tests/test_patroni.py +++ b/tests/test_patroni.py @@ -154,6 +154,7 @@ class TestPatroni(unittest.TestCase): self.p.api.start = Mock() self.p.logger.start = Mock() self.p.config._dynamic_configuration = {} + self.assertRaises(SleepException, self.p.run) with patch('patroni.dcs.Cluster.is_unlocked', Mock(return_value=True)): self.assertRaises(SleepException, self.p.run) with patch('patroni.config.Config.reload_local_configuration', Mock(return_value=False)): diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py index cdd0c160..31454479 100644 --- a/tests/test_postgresql.py +++ b/tests/test_postgresql.py @@ -9,9 +9,9 @@ from mock import Mock, MagicMock, PropertyMock, patch, mock_open import patroni.psycopg as psycopg +from patroni import global_config from patroni.async_executor import CriticalTask from patroni.collections import CaseInsensitiveSet -from patroni.config import GlobalConfig from patroni.dcs import RemoteMember from patroni.exceptions import PostgresConnectionException, PatroniException from patroni.postgresql import Postgresql, STATE_REJECT, STATE_NO_RESPONSE @@ -692,12 +692,12 @@ class TestPostgresql(BaseTestPostgresql): def test_get_server_parameters(self): config = {'parameters': {'wal_level': 'hot_standby', 'max_prepared_transactions': 100}, 'listen': '0'} - self.p._global_config = GlobalConfig({'synchronous_mode': True}) - self.p.config.get_server_parameters(config) - self.p._global_config = GlobalConfig({'synchronous_mode': True, 'synchronous_mode_strict': True}) - self.p.config.get_server_parameters(config) - self.p.config.set_synchronous_standby_names('foo') - self.assertTrue(str(self.p.config.get_server_parameters(config)).startswith('