From 98f50423ca707a57db3595c8e8e06d0a2cddd166 Mon Sep 17 00:00:00 2001 From: Floris van Nee Date: Wed, 2 Sep 2020 13:57:22 +0200 Subject: [PATCH] Add support for configuration directories (#1669) (#1671) It is now also possible to point the configuration path to a directory instead of a file. Patroni will find all yml files in the directory and apply them in sorted order Close https://github.com/zalando/patroni/issues/1669 --- docs/dynamic_configuration.rst | 2 ++ patroni/config.py | 30 ++++++++++++++++++---- tests/test_config.py | 46 +++++++++++++++++++++++++++++++++- 3 files changed, 72 insertions(+), 6 deletions(-) diff --git a/docs/dynamic_configuration.rst b/docs/dynamic_configuration.rst index 4a0e2136..af6cd7df 100644 --- a/docs/dynamic_configuration.rst +++ b/docs/dynamic_configuration.rst @@ -20,6 +20,8 @@ Patroni configuration is stored in the DCS (Distributed Configuration Store). Th It is possible to set/override some of the "Local" configuration parameters with environment variables. Environment configuration is very useful when you are running in a dynamic environment and you don't know some of the parameters in advance (for example it's not possible to know your external IP address when you are running inside ``docker``). +The local configuration can be either a single YAML file or a directory. When it is a directory, all YAML files in that directory are loaded one by one in sorted order. In case a key is defined in multiple files, the occurrence in the last file takes precedence. + Some of the PostgreSQL parameters must hold the same values on the master and the replicas. For those, values set either in the local patroni configuration files or via the environment variables take no effect. To alter or set their values one must change the shared configuration in the DCS. Below is the actual list of such parameters together with the default values: - max_connections: 100 diff --git a/patroni/config.py b/patroni/config.py index b1f0a923..aea866ef 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -92,7 +92,7 @@ class Config(object): self.__environment_configuration = self._build_environment_configuration() # Patroni reads the configuration from the command-line argument if it exists, otherwise from the environment - self._config_file = configfile and os.path.isfile(configfile) and configfile + self._config_file = configfile and os.path.exists(configfile) and configfile if self._config_file: self._local_configuration = self._load_config_file() else: @@ -120,12 +120,32 @@ class Config(object): def check_mode(self, mode): return bool(parse_bool(self._dynamic_configuration.get(mode))) + def _load_config_path(self, path): + """ + If path is a file, loads the yml file pointed to by path. + If path is a directory, loads all yml files in that directory in alphabetical order + """ + if os.path.isfile(path): + files = [path] + elif os.path.isdir(path): + files = [os.path.join(path, f) for f in sorted(os.listdir(path)) + if (f.endswith('.yml') or f.endswith('.yaml')) and os.path.isfile(os.path.join(path, f))] + else: + logger.error('config path %s is neither directory nor file', path) + raise ConfigParseError('invalid config path') + + overall_config = {} + for fname in files: + with open(fname) as f: + config = yaml.safe_load(f) + patch_config(overall_config, config) + return overall_config + def _load_config_file(self): """Loads config.yaml from filesystem and applies some values which were set via ENV""" - with open(self._config_file) as f: - config = yaml.safe_load(f) - patch_config(config, self.__environment_configuration) - return config + config = self._load_config_path(self._config_file) + patch_config(config, self.__environment_configuration) + return config def _load_cache(self): if os.path.isfile(self._cache_file): diff --git a/tests/test_config.py b/tests/test_config.py index 702026df..b656da26 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,9 +1,10 @@ import os import sys import unittest +import io from mock import MagicMock, Mock, patch -from patroni.config import Config +from patroni.config import Config, ConfigParseError from six.moves import builtins @@ -95,3 +96,46 @@ class TestConfig(unittest.TestCase): self.config.set_dynamic_configuration(dynamic_configuration) for name, value in dynamic_configuration['standby_cluster'].items(): self.assertEqual(self.config['standby_cluster'][name], value) + + @patch('os.path.exists', Mock(return_value=True)) + @patch('os.path.isfile', Mock(side_effect=lambda fname: fname != 'postgres0')) + @patch('os.path.isdir', Mock(return_value=True)) + @patch('os.listdir', Mock(return_value=['01-specific.yml', '00-base.yml'])) + def test_configuration_directory(self): + def open_mock(fname, *args, **kwargs): + if fname.endswith('00-base.yml'): + return io.StringIO( + u''' + test: True + test2: + child-1: somestring + child-2: 5 + child-3: False + test3: True + test4: + - abc: 3 + - abc: 4 + ''') + elif fname.endswith('01-specific.yml'): + return io.StringIO( + u''' + test: False + test2: + child-2: 10 + child-3: !!null + test4: + - ab: 5 + new-attr: True + ''') + + with patch.object(builtins, 'open', MagicMock(side_effect=open_mock)): + config = Config('postgres0') + self.assertEqual(config._local_configuration, + {'test': False, 'test2': {'child-1': 'somestring', 'child-2': 10}, + 'test3': True, 'test4': [{'ab': 5}], 'new-attr': True}) + + @patch('os.path.exists', Mock(return_value=True)) + @patch('os.path.isfile', Mock(return_value=False)) + @patch('os.path.isdir', Mock(return_value=False)) + def test_invalid_path(self): + self.assertRaises(ConfigParseError, Config, 'postgres0')