From 4138d0b830e95123035b7e3eea947b7939ab0771 Mon Sep 17 00:00:00 2001 From: Israel Date: Thu, 17 Aug 2023 06:19:49 -0300 Subject: [PATCH] Add docstrings to `patroni.config` (#2708) Besides adding docstrings to `patroni.config`, a few side changes have been applied: * Reference `config_file` property instead of internal attribute `_config_file` in method `_load_config_file`; * Have `_AUTH_ALLOWED_PARAMETERS[:2]` as default value of `params` argument in method `_get_auth` instead of using `params or _AUTH_ALLOWED_PARAMETERS[:2]` in the body; * Use `len(PATRONI_ENV_PREFIX)` instead of a hard-coded `8` when removing the prefix from environment variable names; * Fix documentation of `wal_log_hints` setting. The previous docs mentioned it was a dynamic setting that could be changed. However it is managed by Patroni, which forces `on` value. References: PAT-123. --- docs/patroni_configuration.rst | 2 +- docs/rest_api.rst | 6 +- patroni/config.py | 389 +++++++++++++++++++++++++++------ 3 files changed, 329 insertions(+), 68 deletions(-) diff --git a/docs/patroni_configuration.rst b/docs/patroni_configuration.rst index 559236a2..fafbbbdb 100644 --- a/docs/patroni_configuration.rst +++ b/docs/patroni_configuration.rst @@ -44,7 +44,6 @@ Some of the PostgreSQL parameters **must hold the same values on the primary and - **max_worker_processes**: 8 - **max_prepared_transactions**: 0 - **wal_level**: hot_standby -- **wal_log_hints**: on - **track_commit_timestamp**: off For the parameters below, PostgreSQL does not require equal values among the primary and all the replicas. However, considering the possibility of a replica to become the primary at any time, it doesn't really make sense to set them differently; therefore, **Patroni restricts setting their values to the** :ref:`dynamic configuration `. @@ -62,6 +61,7 @@ There are some other Postgres parameters controlled by Patroni: - **port** - is set either from ``postgresql.listen`` or from ``PATRONI_POSTGRESQL_LISTEN`` environment variable - **cluster_name** - is set either from ``scope`` or from ``PATRONI_SCOPE`` environment variable - **hot_standby: on** +- **wal_log_hints: on** - for Postgres 9.4 and newer. To be on the safe side parameters from the above lists are not written into ``postgresql.conf``, but passed as a list of arguments to the ``pg_ctl start`` which gives them the highest precedence, even above `ALTER SYSTEM `__ diff --git a/docs/rest_api.rst b/docs/rest_api.rst index 9deb3f5e..35ed6fd4 100644 --- a/docs/rest_api.rst +++ b/docs/rest_api.rst @@ -275,7 +275,6 @@ Config endpoint "use_pg_rewind": true, "parameters": { "hot_standby": "on", - "wal_log_hints": "on", "wal_level": "hot_standby", "max_wal_senders": 5, "max_replication_slots": 5, @@ -302,7 +301,6 @@ Config endpoint "use_pg_rewind": true, "parameters": { "hot_standby": "on", - "wal_log_hints": "on", "wal_level": "hot_standby", "max_wal_senders": 5, "max_replication_slots": 5, @@ -355,7 +353,6 @@ If you want to remove (reset) some setting just patch it with ``null``: "hot_standby": "on", "unix_socket_directories": ".", "wal_level": "hot_standby", - "wal_log_hints": "on", "max_wal_senders": 5, "max_replication_slots": 5 } @@ -369,7 +366,7 @@ The above call removes ``postgresql.parameters.max_connections`` from the dynami .. code-block:: bash $ curl -s -XPUT -d \ - '{"maximum_lag_on_failover":1048576,"retry_timeout":10,"postgresql":{"use_slots":true,"use_pg_rewind":true,"parameters":{"hot_standby":"on","wal_log_hints":"on","wal_level":"hot_standby","unix_socket_directories":".","max_wal_senders":5}},"loop_wait":3,"ttl":20}' \ + '{"maximum_lag_on_failover":1048576,"retry_timeout":10,"postgresql":{"use_slots":true,"use_pg_rewind":true,"parameters":{"hot_standby":"on","wal_level":"hot_standby","unix_socket_directories":".","max_wal_senders":5}},"loop_wait":3,"ttl":20}' \ http://localhost:8008/config | jq . { "ttl": 20, @@ -381,7 +378,6 @@ The above call removes ``postgresql.parameters.max_connections`` from the dynami "hot_standby": "on", "unix_socket_directories": ".", "wal_level": "hot_standby", - "wal_log_hints": "on", "max_wal_senders": 5 }, "use_pg_rewind": true diff --git a/patroni/config.py b/patroni/config.py index 4e6da10f..55c8f4bf 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -1,3 +1,4 @@ +"""Facilities related to Patroni configuration.""" import json import logging import os @@ -35,121 +36,162 @@ _AUTH_ALLOWED_PARAMETERS = ( def default_validator(conf: Dict[str, Any]) -> List[str]: + """Ensure *conf* is not empty. + + Designed to be used as default validator for :class:`Config` objects, if no specific validator is provided. + + :param conf: configuration to be validated. + + :returns: an empty list -- :class:`Config` expects the validator to return a list of 0 or more issues found while + validating the configuration. + + :raises: + :class:`ConfigParseError`: if *conf* is empty. + """ if not conf: raise ConfigParseError("Config is empty.") return [] class GlobalConfig(object): + """A class that wraps global configuration and provides convenient methods to access/check values. - """A class that wrapps global configuration and provides convinient methods to access/check values. - - It is instantiated by calling :func:`Config.global_config` method which picks either a - configuration from provided :class:`Cluster` object (the most up-to-date) or from the - local cache if :class::`ClusterConfig` is not initialized or doesn't have a valid config. + It is instantiated either by calling :func:`get_global_config` or :meth:`Config.get_global_config`, which picks + either a configuration from provided :class:`Cluster` object (the most up-to-date) or from the + local cache if :class:`ClusterConfig` is not initialized or doesn't have a valid config. """ def __init__(self, config: Dict[str, Any]) -> None: - """Initialize :class:`GlobalConfig` object. + """Initialize :class:`GlobalConfig` object with given *config*. :param config: current configuration either from - :class:`ClusterConfig` or from :class:`Config.dynamic_configuration` + :class:`ClusterConfig` or from :func:`Config.dynamic_configuration`. """ self.__config = config def get(self, name: str) -> Any: - """Gets global configuration value by name. + """Gets global configuration value by *name*. - :param name: parameter name - :returns: configuration value or `None` if it is missing + :param name: parameter name. + + :returns: configuration value or ``None`` if it is missing. """ return self.__config.get(name) def check_mode(self, mode: str) -> bool: """Checks whether the certain parameter is enabled. - :param mode: parameter name could be: synchronous_mode, failsafe_mode, pause, check_timeline, and so on - :returns: `True` if *mode* is enabled in the global configuration. + :param mode: parameter name, e.g. ``synchronous_mode``, ``failsafe_mode``, ``pause``, ``check_timeline``, and + so on. + + :returns: ``True`` if parameter *mode* is enabled in the global configuration. """ return bool(parse_bool(self.__config.get(mode))) @property def is_paused(self) -> bool: - """:returns: `True` if cluster is in maintenance mode.""" + """``True`` if cluster is in maintenance mode.""" return self.check_mode('pause') @property def is_synchronous_mode(self) -> bool: - """:returns: `True` if synchronous replication is requested.""" + """``True`` if synchronous replication is requested.""" return self.check_mode('synchronous_mode') @property def is_synchronous_mode_strict(self) -> bool: - """:returns: `True` if at least one synchronous node is required.""" + """``True`` if at least one synchronous node is required.""" return self.check_mode('synchronous_mode_strict') def get_standby_cluster_config(self) -> Union[Dict[str, Any], Any]: - """:returns: "standby_cluster" configuration.""" + """Get ``standby_cluster`` configuration. + + :returns: a copy of ``standby_cluster`` configuration. + """ return deepcopy(self.get('standby_cluster')) @property def is_standby_cluster(self) -> bool: - """:returns: `True` if global configuration has a valid "standby_cluster" section.""" + """``True`` if global configuration has a valid ``standby_cluster`` section.""" config = self.get_standby_cluster_config() return isinstance(config, dict) and\ bool(config.get('host') or config.get('port') or config.get('restore_command')) def get_int(self, name: str, default: int = 0) -> int: - """Gets current value from the global configuration and trying to return it as int. + """Gets current value of *name* from the global configuration and try to return it as :class:`int`. - :param name: name of the parameter - :param default: default value if *name* is not in the configuration or invalid - :returns: currently configured value from the global configuration or *default* if it is not set or invalid. + :param name: name of the parameter. + :param default: default value if *name* is not in the configuration or invalid. + + :returns: currently configured value of *name* from the global configuration or *default* if it is not set or + invalid. """ ret = parse_int(self.get(name)) return default if ret is None else ret @property def min_synchronous_nodes(self) -> int: - """:returns: the minimal number of synchronous nodes based on whether strict mode is requested or not.""" + """The minimal number of synchronous nodes based on whether ``synchronous_mode_strict`` is enabled or not.""" return 1 if self.is_synchronous_mode_strict else 0 @property def synchronous_node_count(self) -> int: - """:returns: currently configured value from the global configuration or 1 if it is not set or invalid.""" + """Currently configured value of ``synchronous_node_count`` from the global configuration. + + Assume ``1`` if it is not set or invalid. + """ return max(self.get_int('synchronous_node_count', 1), self.min_synchronous_nodes) @property def maximum_lag_on_failover(self) -> int: - """:returns: currently configured value from the global configuration or 1048576 if it is not set or invalid.""" + """Currently configured value of ``maximum_lag_on_failover`` from the global configuration. + + Assume ``1048576`` if it is not set or invalid. + """ return self.get_int('maximum_lag_on_failover', 1048576) @property def maximum_lag_on_syncnode(self) -> int: - """:returns: currently configured value from the global configuration or -1 if it is not set or invalid.""" + """Currently configured value of ``maximum_lag_on_syncnode`` from the global configuration. + + Assume ``-1`` if it is not set or invalid. + """ return self.get_int('maximum_lag_on_syncnode', -1) @property def primary_start_timeout(self) -> int: - """:returns: currently configured value from the global configuration or 300 if it is not set or invalid.""" + """Currently configured value of ``primary_start_timeout`` from the global configuration. + + Assume ``300`` if it is not set or invalid. + + .. note:: + ``master_start_timeout`` is still supported to keep backward compatibility. + """ default = 300 return self.get_int('primary_start_timeout', default)\ if 'primary_start_timeout' in self.__config else self.get_int('master_start_timeout', default) @property def primary_stop_timeout(self) -> int: - """:returns: currently configured value from the global configuration or 300 if it is not set or invalid.""" + """Currently configured value of ``primary_stop_timeout`` from the global configuration. + + Assume ``0`` if it is not set or invalid. + + .. note:: + ``master_stop_timeout`` is still supported to keep backward compatibility. + """ default = 0 return self.get_int('primary_stop_timeout', default)\ if 'primary_stop_timeout' in self.__config else self.get_int('master_stop_timeout', default) -def get_global_config(cluster: Union[Cluster, None], default: Optional[Dict[str, Any]] = None) -> GlobalConfig: +def get_global_config(cluster: Optional[Cluster], default: Optional[Dict[str, Any]] = None) -> GlobalConfig: """Instantiates :class:`GlobalConfig` based on the input. - :param cluster: the currently known cluster state from DCS - :param default: default configuration, which will be used if there is no valid *cluster.config* - :returns: :class:`GlobalConfig` object + :param cluster: the currently known cluster state from DCS. + :param default: default configuration, which will be used if there is no valid *cluster.config*. + + :returns: :class:`GlobalConfig` object. """ # Try to protect from the case when DCS was wiped out if cluster and cluster.config and cluster.config.modify_version: @@ -160,23 +202,29 @@ def get_global_config(cluster: Union[Cluster, None], default: Optional[Dict[str, class Config(object): - """ + """Handle Patroni configuration. + This class is responsible for: - 1) Building and giving access to `effective_configuration` from: - * `Config.__DEFAULT_CONFIG` -- some sane default values - * `dynamic_configuration` -- configuration stored in DCS - * `local_configuration` -- configuration from `config.yml` or environment + 1) Building and giving access to ``effective_configuration`` from: - 2) Saving and loading `dynamic_configuration` into 'patroni.dynamic.json' file + * ``Config.__DEFAULT_CONFIG`` -- some sane default values; + * ``dynamic_configuration`` -- configuration stored in DCS; + * ``local_configuration`` -- configuration from `config.yml` or environment. + + 2) Saving and loading ``dynamic_configuration`` into 'patroni.dynamic.json' file located in local_configuration['postgresql']['data_dir'] directory. - This is necessary to be able to restore `dynamic_configuration` - if DCS was accidentally wiped + This is necessary to be able to restore ``dynamic_configuration`` + if DCS was accidentally wiped. - 3) Loading of configuration file in the old format and converting it into new format + 3) Loading of configuration file in the old format and converting it into new format. - 4) Mimicking some of the `dict` interfaces to make it possible - to work with it as with the old `config` object. + 4) Mimicking some ``dict`` interfaces to make it possible + to work with it as with the old ``config`` object. + + :cvar PATRONI_CONFIG_VARIABLE: name of the environment variable that can be used to load Patroni configuration from. + :cvar __CACHE_FILENAME: name of the file used to cache dynamic configuration under Postgres data directory. + :cvar __DEFAULT_CONFIG: default configuration values for some Patroni settings. """ PATRONI_CONFIG_VARIABLE = PATRONI_ENV_PREFIX + 'CONFIGURATION' @@ -202,12 +250,30 @@ class Config(object): def __init__(self, configfile: str, validator: Optional[Callable[[Dict[str, Any]], List[str]]] = default_validator) -> None: + """Create a new instance of :class:`Config` and validate the loaded configuration using *validator*. + + .. note:: + Patroni will read configuration from these locations in this order: + + * file or directory path passed as command-line argument (*configfile*), if it exists and the file or + files found in the directory can be parsed (see :meth:`~Config._load_config_path`), otherwise + * YAML file passed via the environment variable (see :cvar:`PATRONI_CONFIG_VARIABLE`), if the referenced + file exists and can be parsed, otherwise + * from configuration values defined as environment variables, see + :meth:`~Config._build_environment_configuration`. + + :param configfile: path to Patroni configuration file. + :param validator: function used to validate Patroni configuration. It should receive a dictionary which + represents Patroni configuration, and return a list of zero or more error messages based on validation. + + :raises: + :class:`ConfigParseError`: if any issue is reported by *validator*. + """ self._modify_version = -1 self._dynamic_configuration = {} self.__environment_configuration = self._build_environment_configuration() - # Patroni reads the configuration from the command-line argument if it exists, otherwise from the environment self._config_file = configfile if configfile and os.path.exists(configfile) else None if self._config_file: self._local_configuration = self._load_config_file() @@ -228,11 +294,13 @@ class Config(object): self._cache_needs_saving = False @property - def config_file(self) -> Union[str, None]: + def config_file(self) -> Optional[str]: + """Path to Patroni configuration file, if any, else ``None``.""" return self._config_file @property def dynamic_configuration(self) -> Dict[str, Any]: + """Deep copy of cached Patroni dynamic configuration.""" return deepcopy(self._dynamic_configuration) @property @@ -252,9 +320,17 @@ class Config(object): return deepcopy(cls.__DEFAULT_CONFIG) def _load_config_path(self, path: str) -> Dict[str, Any]: - """ - If path is a file, loads the yml file pointed to by path. - If path is a directory, loads all yml files in that directory in alphabetical order + """Load Patroni configuration file(s) from *path*. + + If *path* is a file, load the yml file pointed to by *path*. + If *path* is a directory, load all yml files in that directory in alphabetical order. + + :param path: path to either an YAML configuration file, or to a folder containing YAML configuration files. + + :returns: configuration after reading the configuration file(s) from *path*. + + :raises: + :class:`ConfigParseError`: if *path* is invalid. """ if os.path.isfile(path): files = [path] @@ -273,14 +349,18 @@ class Config(object): return overall_config def _load_config_file(self) -> Dict[str, Any]: - """Loads config.yaml from filesystem and applies some values which were set via ENV""" + """Load configuration file(s) from filesystem and apply values which were set via environment variables. + + :returns: final configuration after merging configuration file(s) and environment variables. + """ if TYPE_CHECKING: # pragma: no cover - assert self._config_file is not None - config = self._load_config_path(self._config_file) + assert self.config_file is not None + config = self._load_config_path(self.config_file) patch_config(config, self.__environment_configuration) return config def _load_cache(self) -> None: + """Load dynamic configuration from ``patroni.dynamic.json``.""" if os.path.isfile(self._cache_file): try: with open(self._cache_file) as f: @@ -289,6 +369,12 @@ class Config(object): logger.exception('Exception when loading file: %s', self._cache_file) def save_cache(self) -> None: + """Save dynamic configuration to ``patroni.dynamic.json`` under Postgres data directory. + + .. note:: + ``patroni.dynamic.jsonXXXXXX`` is created as a temporary file and than renamed to ``patroni.dynamic.json``, + where ``XXXXXX`` is a random suffix. + """ if self._cache_needs_saving: tmpfile = fd = None try: @@ -315,9 +401,16 @@ class Config(object): # configuration could be either ClusterConfig or dict def set_dynamic_configuration(self, configuration: Union[ClusterConfig, Dict[str, Any]]) -> bool: + """Set dynamic configuration values with given *configuration*. + + :param configuration: new dynamic configuration values. Supports :class:`dict` for backward compatibility. + + :returns: ``True`` if changes have been detected between current dynamic configuration and the new dynamic + *configuration*, ``False`` otherwise. + """ if isinstance(configuration, ClusterConfig): if self._modify_version == configuration.modify_version: - return False # If the version didn't changed there is nothing to do + return False # If the version didn't change there is nothing to do self._modify_version = configuration.modify_version configuration = configuration.data @@ -333,6 +426,14 @@ class Config(object): return False def reload_local_configuration(self) -> Optional[bool]: + """Reload configuration values from the configuration file(s). + + .. note:: + Designed to be used when user applies changes to configuration file(s), so Patroni can use the new values + with a reload instead of a restart. + + :returns: ``True`` if changes have been detected between current local configuration + """ if self.config_file: try: configuration = self._load_config_file() @@ -348,6 +449,38 @@ class Config(object): @staticmethod def _process_postgresql_parameters(parameters: Dict[str, Any], is_local: bool = False) -> Dict[str, Any]: + """Process Postgres *parameters*. + + .. note:: + If *is_local* configuration discard any setting from *parameters* that is listed under + :attr:`~patroni.postgresql.config.ConfigHandler.CMDLINE_OPTIONS` as those are supposed to be set only + through dynamic configuration. + + When setting parameters from :attr:`~patroni.postgresql.config.ConfigHandler.CMDLINE_OPTIONS` through + dynamic configuration their value will be validated as per the validator defined in that very same + attribute entry. If the given value cannot be validated, a warning will be logged and the default value of + the GUC will be used instead. + + Some parameters from :attr:`~patroni.postgresql.config.ConfigHandler.CMDLINE_OPTIONS` cannot be set even if + not *is_local* configuration: + + * ``listen_addresses``: inferred from ``postgresql.listen`` local configuration or from + ``PATRONI_POSTGRESQL_LISTEN`` environment variable; + * ``port``: inferred from ``postgresql.listen`` local configuration or from + ``PATRONI_POSTGRESQL_LISTEN`` environment variable; + * ``cluster_name``: set through ``scope`` local configuration or through ``PATRONI_SCOPE`` environment + variable; + * ``hot_standby``: always enabled; + * ``wal_log_hints``: always enabled. + + :param parameters: Postgres parameters to be processed. Should be the parsed YAML value of + ``postgresql.parameters`` configuration, either from local or from dynamic configuration. + + :param is_local: should be ``True`` if *parameters* refers to local configuration, or ``False`` if *parameters* + refers to dynamic configuration. + + :returns: new value for ``postgresql.parameters`` after processing and validating *parameters*. + """ pg_params: Dict[str, Any] = {} for name, value in (parameters or {}).items(): @@ -363,6 +496,32 @@ class Config(object): return pg_params def _safe_copy_dynamic_configuration(self, dynamic_configuration: Dict[str, Any]) -> Dict[str, Any]: + """Create a copy of *dynamic_configuration*. + + Merge *dynamic_configuration* with :attr:`__DEFAULT_CONFIG` (*dynamic_configuration* takes precedence), and + process ``postgresql.parameters`` from *dynamic_configuration* through :func:`_process_postgresql_parameters`, + if present. + + .. note:: + The following settings are not allowed in ``postgresql`` section as they are intended to be local + configuration, and are removed if present: + + * ``connect_address``; + * ``proxy_address``; + * ``listen``; + * ``config_dir``; + * ``data_dir``; + * ``pgpass``; + * ``authentication``; + + Besides that any setting present in *dynamic_configuration* but absent from :attr:`__DEFAULT_CONFIG` is + discarded. + + :param dynamic_configuration: Patroni dynamic configuration. + + :returns: copy of *dynamic_configuration*, merged with default dynamic configuration and with some sanity checks + performed over it. + """ config = self.get_default_config() for name, value in dynamic_configuration.items(): @@ -383,9 +542,25 @@ class Config(object): @staticmethod def _build_environment_configuration() -> Dict[str, Any]: + """Get local configuration settings that were specified through environment variables. + + :returns: dictionary containing the found environment variables and their values, respecting the expected + structure of Patroni configuration. + """ ret: Dict[str, Any] = defaultdict(dict) - def _popenv(name: str) -> Union[str, None]: + def _popenv(name: str) -> Optional[str]: + """Get value of environment variable *name*. + + .. note:: + *name* is prefixed with :data:`~patroni.PATRONI_ENV_PREFIX` when searching in the environment. + + Also, the corresponding environment variable is removed from the environment upon reading its value. + + :param name: name of the environment variable. + + :returns: value of *name*, if present in the environment, otherwise ``None``. + """ return os.environ.pop(PATRONI_ENV_PREFIX + name.upper(), None) for param in ('name', 'namespace', 'scope'): @@ -394,6 +569,23 @@ class Config(object): ret[param] = value def _fix_log_env(name: str, oldname: str) -> None: + """Normalize a log related environment variable. + + .. note:: + Patroni used to support different names for log related environment variables in the past. As the + environment variables were renamed, this function takes care of mapping and normalizing the environment. + + *name* is prefixed with :data:`~patroni.PATRONI_ENV_PREFIX` and ``LOG`` when searching in the + environment. + + *oldname* is prefixed with :data:`~patroni.PATRONI_ENV_PREFIX` when searching in the environment. + + If both *name* and *oldname* are set in the environment, *name* takes precedence. + + :param name: new name of a log related environment variable. + :param oldname: original name of a log related environment variable. + :type oldname: str + """ value = _popenv(oldname) name = PATRONI_ENV_PREFIX + 'LOG_' + name.upper() if value and name not in os.environ: @@ -403,6 +595,15 @@ class Config(object): _fix_log_env(name, oldname) def _set_section_values(section: str, params: List[str]) -> None: + """Get value of *params* environment variables that are related with *section*. + + .. note:: + The values are retrieved from the environment and updated directly into the returning dictionary of + :func:`_build_environment_configuration`. + + :param section: configuration section the *params* belong to. + :param params: name of the Patroni settings. + """ for param in params: value = _popenv(section + '_' + param) if value: @@ -424,6 +625,7 @@ class Config(object): if value: ret['postgresql'].setdefault('bin_name', {})[binary] = value + # parse all values retrieved from the environment as Python objects, according to the expected type for first, second in (('restapi', 'allowlist_include_members'), ('ctl', 'insecure')): value = ret.get(first, {}).pop(second, None) if value: @@ -440,7 +642,13 @@ class Config(object): if value is not None: ret[first][second] = value - def _parse_list(value: str) -> Union[List[str], None]: + def _parse_list(value: str) -> Optional[List[str]]: + """Parse an YAML list *value* as a :class:`list`. + + :param value: YAML list as a string. + + :returns: *value* as :class:`list`. + """ if not (value.strip().startswith('-') or '[' in value): value = '[{0}]'.format(value) try: @@ -456,7 +664,13 @@ class Config(object): if value: ret[first][second] = value - def _parse_dict(value: str) -> Union[Dict[str, Any], None]: + def _parse_dict(value: str) -> Optional[Dict[str, Any]]: + """Parse an YAML dictionary *value* as a :class:`dict`. + + :param value: YAML dictionary as a string. + + :returns: *value* as :class:`dict`. + """ if not value.strip().startswith('{'): value = '{{{0}}}'.format(value) try: @@ -473,9 +687,16 @@ class Config(object): if value: ret[first][second] = value - def _get_auth(name: str, params: Optional[Collection[str]] = None) -> Dict[str, str]: + def _get_auth(name: str, params: Collection[str] = _AUTH_ALLOWED_PARAMETERS[:2]) -> Dict[str, str]: + """Get authorization related environment variables *params* from section *name*. + + :param name: name of a configuration section that may contain authorization *params*. + :param params: the authorization settings that may be set under section *name*. + + :returns: dictionary containing environment values for authorization *params* of section *name*. + """ ret: Dict[str, str] = {} - for param in params or _AUTH_ALLOWED_PARAMETERS[:2]: + for param in params: value = _popenv(name + '_' + param) if value: ret[param] = value @@ -498,7 +719,7 @@ class Config(object): for param in list(os.environ.keys()): if param.startswith(PATRONI_ENV_PREFIX): # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..) - name, suffix = (param[8:].split('_', 1) + [''])[:2] + name, suffix = (param[len(PATRONI_ENV_PREFIX):].split('_', 1) + [''])[:2] if suffix in ('HOST', 'HOSTS', 'PORT', 'USE_PROXIES', 'PROTOCOL', 'SRV', 'SRV_SUFFIX', 'URL', 'PROXY', 'CACERT', 'CERT', 'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC', 'CONSISTENCY', 'REGISTER_SERVICE', 'SERVICE_CHECK_INTERVAL', 'SERVICE_CHECK_TLS_SERVER_NAME', @@ -529,14 +750,14 @@ class Config(object): users = {} for param in list(os.environ.keys()): if param.startswith(PATRONI_ENV_PREFIX): - name, suffix = (param[8:].rsplit('_', 1) + [''])[:2] + name, suffix = (param[len(PATRONI_ENV_PREFIX):].rsplit('_', 1) + [''])[:2] # PATRONI__PASSWORD=, PATRONI__OPTIONS= # CREATE USER "" WITH PASSWORD '' if name and suffix == 'PASSWORD': password = os.environ.pop(param) if password: users[name] = {'password': password} - options = os.environ.pop(param[:-9] + '_OPTIONS', None) + options = os.environ.pop(param[:-9] + '_OPTIONS', None) # replace "_PASSWORD" with "_OPTIONS" options = options and _parse_list(options) if options: users[name]['options'] = options @@ -547,6 +768,16 @@ class Config(object): def _build_effective_configuration(self, dynamic_configuration: Dict[str, Any], local_configuration: Dict[str, Union[Dict[str, Any], Any]]) -> Dict[str, Any]: + """Build effective configuration by merging *dynamic_configuration* and *local_configuration*. + + .. note:: + *local_configuration* takes precedence over *dynamic_configuration* if a setting is defined in both. + + :param dynamic_configuration: Patroni dynamic configuration. + :param local_configuration: Patroni local configuration. + + :returns: _description_ + """ config = self._safe_copy_dynamic_configuration(dynamic_configuration) for name, value in local_configuration.items(): if name == 'citus': # remove invalid citus configuration @@ -610,23 +841,57 @@ class Config(object): return config def get(self, key: str, default: Optional[Any] = None) -> Any: + """Get effective value of ``key`` setting from Patroni configuration root. + + Designed to work the same way as :func:`dict.get`. + + :param key: name of the setting. + :param default: default value if *key* is not present in the effective configuration. + + :returns: value of *key*, if present in the effective configuration, otherwise *default*. + """ return self.__effective_configuration.get(key, default) def __contains__(self, key: str) -> bool: + """Check if setting *key* is present in the effective configuration. + + Designed to work the same way as :func:`dict.__contains__`. + + :param key: name of the setting to be checked. + + :returns: ``True`` if setting *key* exists in effective configuration, else ``False``. + """ return key in self.__effective_configuration def __getitem__(self, key: str) -> Any: + """Get value of setting *key* from effective configuration. + + Designed to work the same way as :func:`dict.__getitem__`. + + :param key: name of the setting. + + :returns: value of setting *key*. + + :raises: + :class:`KeyError`: if *key* is not present in effective configuration. + """ return self.__effective_configuration[key] def copy(self) -> Dict[str, Any]: + """Get a deep copy of effective Patroni configuration. + + :returns: a deep copy of the Patroni configuration. + """ return deepcopy(self.__effective_configuration) - def get_global_config(self, cluster: Union[Cluster, None]) -> GlobalConfig: + def get_global_config(self, cluster: Optional[Cluster]) -> GlobalConfig: """Instantiate :class:`GlobalConfig` based on input. Use the configuration from provided *cluster* (the most up-to-date) or from the local cache if *cluster.config* is not initialized or doesn't have a valid config. - :param cluster: the currently known cluster state from DCS - :returns: :class:`GlobalConfig` object + + :param cluster: the currently known cluster state from DCS. + + :returns: :class:`GlobalConfig` object. """ return get_global_config(cluster, self._dynamic_configuration)