Add support for configuration directories (#1669) (#1671)

It is now also possible to point the configuration path to a directory instead of a file.
Patroni will find all yml files in the directory and apply them in sorted order

Close https://github.com/zalando/patroni/issues/1669

docs/dynamic_configuration.rst

@@ -20,6 +20,8 @@ Patroni configuration is stored in the DCS (Distributed Configuration Store). Th
 	It is possible to set/override some of the "Local" configuration parameters with environment variables.
 	Environment configuration is very useful when you are running in a dynamic environment and you don't know some of the parameters in advance (for example it's not possible to know your external IP address when you are running inside ``docker``).
 
+The local configuration can be either a single YAML file or a directory. When it is a directory, all YAML files in that directory are loaded one by one in sorted order. In case a key is defined in multiple files, the occurrence in the last file takes precedence.
+
 Some of the PostgreSQL parameters must hold the same values on the master and the replicas. For those, values set either in the local patroni configuration files or via the environment variables take no effect. To alter or set their values one must change the shared configuration in the DCS. Below is the actual list of such parameters together with the default values:
 
 - max_connections: 100

patroni/config.py

@@ -92,7 +92,7 @@ class Config(object):
         self.__environment_configuration = self._build_environment_configuration()
 
         # Patroni reads the configuration from the command-line argument if it exists, otherwise from the environment
-        self._config_file = configfile and os.path.isfile(configfile) and configfile
+        self._config_file = configfile and os.path.exists(configfile) and configfile
         if self._config_file:
             self._local_configuration = self._load_config_file()
         else:
@@ -120,12 +120,32 @@ class Config(object):
     def check_mode(self, mode):
         return bool(parse_bool(self._dynamic_configuration.get(mode)))
 
+    def _load_config_path(self, path):
+        """
+        If path is a file, loads the yml file pointed to by path.
+        If path is a directory, loads all yml files in that directory in alphabetical order
+        """
+        if os.path.isfile(path):
+            files = [path]
+        elif os.path.isdir(path):
+            files = [os.path.join(path, f) for f in sorted(os.listdir(path))
+                     if (f.endswith('.yml') or f.endswith('.yaml')) and os.path.isfile(os.path.join(path, f))]
+        else:
+            logger.error('config path %s is neither directory nor file', path)
+            raise ConfigParseError('invalid config path')
+
+        overall_config = {}
+        for fname in files:
+            with open(fname) as f:
+                config = yaml.safe_load(f)
+                patch_config(overall_config, config)
+        return overall_config
+
     def _load_config_file(self):
         """Loads config.yaml from filesystem and applies some values which were set via ENV"""
-        with open(self._config_file) as f:
-            config = yaml.safe_load(f)
-            patch_config(config, self.__environment_configuration)
-            return config
+        config = self._load_config_path(self._config_file)
+        patch_config(config, self.__environment_configuration)
+        return config
 
     def _load_cache(self):
         if os.path.isfile(self._cache_file):

tests/test_config.py

@@ -1,9 +1,10 @@
 import os
 import sys
 import unittest
+import io
 
 from mock import MagicMock, Mock, patch
-from patroni.config import Config
+from patroni.config import Config, ConfigParseError
 from six.moves import builtins
 
 
@@ -95,3 +96,46 @@ class TestConfig(unittest.TestCase):
         self.config.set_dynamic_configuration(dynamic_configuration)
         for name, value in dynamic_configuration['standby_cluster'].items():
             self.assertEqual(self.config['standby_cluster'][name], value)
+
+    @patch('os.path.exists', Mock(return_value=True))
+    @patch('os.path.isfile', Mock(side_effect=lambda fname: fname != 'postgres0'))
+    @patch('os.path.isdir', Mock(return_value=True))
+    @patch('os.listdir', Mock(return_value=['01-specific.yml', '00-base.yml']))
+    def test_configuration_directory(self):
+        def open_mock(fname, *args, **kwargs):
+            if fname.endswith('00-base.yml'):
+                return io.StringIO(
+                    u'''
+                    test: True
+                    test2:
+                      child-1: somestring
+                      child-2: 5
+                      child-3: False
+                    test3: True
+                    test4:
+                     - abc: 3
+                     - abc: 4
+                    ''')
+            elif fname.endswith('01-specific.yml'):
+                return io.StringIO(
+                    u'''
+                    test: False
+                    test2:
+                      child-2: 10
+                      child-3: !!null
+                    test4:
+                     - ab: 5
+                    new-attr: True
+                    ''')
+
+        with patch.object(builtins, 'open', MagicMock(side_effect=open_mock)):
+            config = Config('postgres0')
+            self.assertEqual(config._local_configuration,
+                             {'test': False, 'test2': {'child-1': 'somestring', 'child-2': 10},
+                              'test3': True, 'test4': [{'ab': 5}], 'new-attr': True})
+
+    @patch('os.path.exists', Mock(return_value=True))
+    @patch('os.path.isfile', Mock(return_value=False))
+    @patch('os.path.isdir', Mock(return_value=False))
+    def test_invalid_path(self):
+        self.assertRaises(ConfigParseError, Config, 'postgres0')