diff --git a/patroni/postgresql/available_parameters/__init__.py b/patroni/postgresql/available_parameters/__init__.py new file mode 100644 index 00000000..8af63cd9 --- /dev/null +++ b/patroni/postgresql/available_parameters/__init__.py @@ -0,0 +1,58 @@ +import logging +import sys + +from pathlib import Path +from typing import Iterator + +logger = logging.getLogger(__name__) + +if sys.version_info < (3, 9): + PathLikeObj = Path + conf_dir = Path(__file__).parent +else: + from importlib.resources import files + + if sys.version_info < (3, 11): # pragma: no cover + from importlib.abc import Traversable + else: # pragma: no cover + from importlib.resources.abc import Traversable + + PathLikeObj = Traversable + conf_dir = files(__name__) + + +def get_validator_files() -> Iterator[PathLikeObj]: + """Recursively find YAML files from the current package directory. + + :returns: an iterator of :class:`PathLikeObj` objects representing validator files. + """ + return _traversable_walk(conf_dir.iterdir()) + + +def _traversable_walk(tvbs: Iterator[PathLikeObj]) -> Iterator[PathLikeObj]: + """Recursively walk through Path/Traversable objects, yielding all YAML files in deterministic order. + + :param tvbs: An iterator over :class:`PathLikeObj` objects, where each object is a file or directory + that potentially contains YAML files. + + :yields: :class:`PathLikeObj` objects representing YAML files found during the traversal. + """ + for tvb in _filter_and_sort_files(tvbs): + if tvb.is_file(): + yield tvb + elif tvb.is_dir(): + yield from _traversable_walk(tvb.iterdir()) + + +def _filter_and_sort_files(files: Iterator[PathLikeObj]) -> Iterator[PathLikeObj]: + """Sort files by name, and filter out non-YAML files and Python files. + + :param files: A list of files and/or directories to be filtered and sorted. + + :yields: filtered and sorted objects. + """ + for file in sorted(files, key=lambda x: x.name): + if file.name.lower().endswith((".yml", ".yaml")) or file.is_dir(): + yield file + elif not file.name.lower().endswith((".py", ".pyc")): + logger.info("Ignored a non-YAML file found under `%s` directory: `%s`.", __name__.split('.')[-1], file) diff --git a/patroni/postgresql/validator.py b/patroni/postgresql/validator.py index 384db08d..f118c973 100644 --- a/patroni/postgresql/validator.py +++ b/patroni/postgresql/validator.py @@ -1,11 +1,11 @@ import abc from copy import deepcopy import logging -import os import yaml from typing import Any, Dict, Iterator, List, MutableMapping, Optional, Tuple, Type, Union +from .available_parameters import get_validator_files, PathLikeObj from ..collections import CaseInsensitiveDict, CaseInsensitiveSet from ..exceptions import PatroniException from ..utils import parse_bool, parse_int, parse_real @@ -258,10 +258,10 @@ class InvalidGucValidatorsFile(PatroniException): """Raised when reading or parsing of a YAML file faces an issue.""" -def _read_postgres_gucs_validators_file(file: str) -> Dict[str, Any]: +def _read_postgres_gucs_validators_file(file: PathLikeObj) -> Dict[str, Any]: """Read an YAML file and return the corresponding Python object. - :param file: path to the file to be read. It is expected to be encoded with ``UTF-8``, and to be a YAML document. + :param file: path-like object to read from. It is expected to be encoded with ``UTF-8``, and to be a YAML document. :returns: the YAML content parsed into a Python object. If any issue is faced while reading/parsing the file, then return ``None``. @@ -270,7 +270,7 @@ def _read_postgres_gucs_validators_file(file: str) -> Dict[str, Any]: :class:`InvalidGucValidatorsFile`: if faces an issue while reading or parsing *file*. """ try: - with open(file, encoding='UTF-8') as stream: + with file.open(encoding='UTF-8') as stream: return yaml.safe_load(stream) except Exception as exc: raise InvalidGucValidatorsFile( @@ -385,21 +385,7 @@ def _load_postgres_gucs_validators() -> None: version_till: null """ - conf_dir = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - 'available_parameters', - ) - yaml_files: List[str] = [] - - for root, _, files in os.walk(conf_dir): - for file in sorted(files): - full_path = os.path.join(root, file) - if file.lower().endswith(('.yml', '.yaml')): - yaml_files.append(full_path) - else: - logger.info('Ignored a non-YAML file found under `available_parameters` directory: `%s`.', full_path) - - for file in yaml_files: + for file in get_validator_files(): try: config: Dict[str, Any] = _read_postgres_gucs_validators_file(file) except InvalidGucValidatorsFile as exc: diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py index ef60b9b0..b6ebc3de 100644 --- a/tests/test_postgresql.py +++ b/tests/test_postgresql.py @@ -7,6 +7,7 @@ import time from copy import deepcopy from mock import Mock, MagicMock, PropertyMock, patch, mock_open +from pathlib import Path import patroni.psycopg as psycopg @@ -1064,7 +1065,7 @@ class TestPostgresql(BaseTestPostgresql): def test__read_postgres_gucs_validators_file(self): # raise exception with self.assertRaises(InvalidGucValidatorsFile) as exc: - _read_postgres_gucs_validators_file('random_file.yaml') + _read_postgres_gucs_validators_file(Path('random_file.yaml')) self.assertEqual( str(exc.exception), "Unexpected issue while reading parameters file `random_file.yaml`: `[Errno 2] No such file or directory: " @@ -1073,17 +1074,32 @@ class TestPostgresql(BaseTestPostgresql): def test__load_postgres_gucs_validators(self): # log messages - with patch('os.walk', Mock(return_value=iter([('.', [], ['file.txt', 'random.yaml'])]))), \ - patch('patroni.postgresql.validator.logger.info') as mock_info, \ + file1_attrs = {'is_file.return_value': True, 'is_dir.return_value': False} + file1_mock = MagicMock(**file1_attrs) + file1_mock.name = '__init__.py' + file2_attrs = {'is_file.return_value': False, 'is_dir.return_value': True, 'iterdir.return_value': []} + file2_mock = MagicMock(**file2_attrs) + file2_mock.name = '__pycache__' + file3_attrs = {'is_file.return_value': True, 'is_dir.return_value': False} + file3_mock = MagicMock(**file3_attrs) + file3_mock.name = file3_mock.__str__.return_value = 'random.yaml' + file3_mock.open.side_effect = FileNotFoundError('[Errno 2] No such file or directory: random.yaml') + file4_attrs = {'is_file.return_value': True, 'is_dir.return_value': False} + file4_mock = MagicMock(**file4_attrs) + file4_mock.name = 'file.txt' + dir_attrs = {'name': 'available_parameters', 'is_file.return_value': False, 'is_dir.return_value': True} + dir_mock = MagicMock(**dir_attrs) + dir_mock.iterdir.return_value = [file1_mock, file2_mock, file3_mock, file4_mock] + with patch('patroni.postgresql.available_parameters.conf_dir', dir_mock), \ + patch('patroni.postgresql.available_parameters.logger.info') as mock_info, \ patch('patroni.postgresql.validator.logger.warning') as mock_warning: _load_postgres_gucs_validators() - mock_info.assert_called_once_with('Ignored a non-YAML file found under `available_parameters` directory: ' - '`%s`.', os.path.join('.', 'file.txt')) + mock_info.assert_called_once_with('Ignored a non-YAML file found under `%s` ' + 'directory: `%s`.', 'available_parameters', file4_mock) mock_warning.assert_called_once() self.assertIn( - "Unexpected issue while reading parameters file `{0}`: `[Errno 2] No such file or " - "directory:".format(os.path.join('.', 'random.yaml')), - mock_warning.call_args[0][0] + "Unexpected issue while reading parameters file `random.yaml`: `[Errno 2] No such file or " + "directory:", mock_warning.call_args[0][0] )