Make it possible to work without config.yml

Most of the basic configuration could be done via ENV

patroni/__init__.py

@@ -1,7 +1,5 @@
 import logging
-import os
 import signal
-import sys
 import time
 
 from patroni.api import RestApiServer
@@ -17,11 +15,10 @@ logger = logging.getLogger(__name__)
 
 
 class Patroni(object):
-    PATRONI_CONFIG_VARIABLE = 'PATRONI_CONFIGURATION'
 
-    def __init__(self, config_file=None, config_env=None):
+    def __init__(self):
         self.version = __version__
-        self.config = Config(config_file=config_file, config_env=config_env)
+        self.config = Config()
         self.dcs = get_dcs(self.config)
         self.load_dynamic_configuration()
 
@@ -116,18 +113,7 @@ def main():
     logging.getLogger('requests').setLevel(logging.WARNING)
     setup_signal_handlers()
 
-    # Patroni reads the configuration from the command-line argument if it exists, and from the environment otherwise.
+    patroni = Patroni()
-    config_env = False
-    config_file = len(sys.argv) >= 2 and os.path.isfile(sys.argv[1]) and sys.argv[1]
-    if not config_file:
-        config_env = os.environ.pop(Patroni.PATRONI_CONFIG_VARIABLE, None)
-        if config_env is None:
-            print('Usage: {0} config.yml'.format(sys.argv[0]))
-            print('\tPatroni may also read the configuration from the {} environment variable'.
-                  format(Patroni.PATRONI_CONFIG_VARIABLE))
-            return
-
-    patroni = Patroni(config_file, config_env)
     try:
         patroni.run()
     except KeyboardInterrupt:

patroni/config.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import os
+import sys
 import tempfile
 import yaml
 
@@ -33,6 +34,9 @@ class Config(object):
          to work with it as with the old `config` object.
     """
 
+    PATRONI_ENV_PREFIX = 'PATRONI_'
+    PATRONI_CONFIG_VARIABLE = PATRONI_ENV_PREFIX + 'CONFIGURATION'
+
     __CACHE_FILENAME = 'patroni.dynamic.json'
     __DEFAULT_CONFIG = {
         'ttl': 30, 'loop_wait': 10, 'retry_timeout': 10,
@@ -42,15 +46,25 @@ class Config(object):
         }
     }
 
-    def __init__(self, config_file=None, config_env=None):
+    def __init__(self):
-        self._config_file = None if config_env else config_file
         self._modify_index = -1
         self._dynamic_configuration = {}
-        if config_env:
+
-            self._local_configuration = yaml.safe_load(config_env)
+        self.__environment_configuration = self._build_environment_configuration()
-        else:
+
-            self.__environment_configuration = self._build_environment_configuration()
+        # Patroni reads the configuration from the command-line argument if it exists, otherwise from the environment
+        self._config_file = len(sys.argv) >= 2 and os.path.isfile(sys.argv[1]) and sys.argv[1]
+        if self._config_file:
             self._local_configuration = self._load_config_file()
+        else:
+            config_env = os.environ.pop(self.PATRONI_CONFIG_VARIABLE, None)
+            self._local_configuration = config_env and yaml.safe_load(config_env) or self.__environment_configuration
+            if not self._local_configuration:
+                print('Usage: {0} config.yml'.format(sys.argv[0]))
+                print('\tPatroni may also read the configuration from the {0} environment variable'.
+                      format(self.PATRONI_CONFIG_VARIABLE))
+                exit(1)
+
         self.__effective_configuration = self._build_effective_configuration(self._dynamic_configuration,
                                                                              self._local_configuration)
         self._data_dir = self.__effective_configuration['postgresql']['data_dir']
@@ -168,7 +182,7 @@ class Config(object):
         ret = defaultdict(dict)
 
         def _popenv(name):
-            return os.environ.pop('PATRONI_' + name.upper(), None)
+            return os.environ.pop(Config.PATRONI_ENV_PREFIX + name.upper(), None)
 
         for param in ('name', 'namespace', 'scope'):
             value = _popenv(param)
@@ -216,7 +230,7 @@ class Config(object):
                 return None
 
         for param in list(os.environ.keys()):
-            if param.startswith('PATRONI_'):
+            if param.startswith(Config.PATRONI_ENV_PREFIX):
                 name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
                 if name and suffix:
                     # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT)

tests/test_config.py

@@ -1,5 +1,6 @@
 import os
 import unittest
+import sys
 
 from mock import MagicMock, Mock, patch
 from patroni.config import Config
@@ -12,7 +13,12 @@ class TestConfig(unittest.TestCase):
     @patch('json.load', Mock(side_effect=Exception))
     @patch.object(builtins, 'open', MagicMock())
     def setUp(self):
-        self.config = Config(config_env='restapi: {}\npostgresql: {data_dir: foo}')
+        sys.argv = ['patroni.py']
+        os.environ[Config.PATRONI_CONFIG_VARIABLE] = 'restapi: {}\npostgresql: {data_dir: foo}'
+        self.config = Config()
+
+    def test_no_config(self):
+        self.assertRaises(SystemExit, Config)
 
     @patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception))
     def test_set_dynamic_configuration(self):
@@ -46,7 +52,8 @@ class TestConfig(unittest.TestCase):
             'PATRONI_admin_PASSWORD': 'admin',
             'PATRONI_admin_OPTIONS': 'createrole,createdb'
         })
-        config = Config(config_file='postgres0.yml')
+        sys.argv = ['patroni.py', 'postgres0.yml']
+        config = Config()
         with patch.object(Config, '_load_config_file', Mock(return_value={'restapi': {}})):
             with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
                 self.assertRaises(Exception, config.reload_local_configuration, True)

tests/test_ha.py

@@ -1,7 +1,8 @@
-import etcd
-import unittest
 import datetime
+import etcd
+import os
 import pytz
+import unittest
 
 from mock import Mock, MagicMock, patch
 from patroni.config import Config
@@ -49,7 +50,7 @@ def get_cluster_initialized_with_only_leader(failover=None):
 class MockPatroni(object):
 
     def __init__(self, p, d):
-        self.config = Config(config_env="""
+        os.environ[Config.PATRONI_CONFIG_VARIABLE] = """
 restapi:
   listen: 0.0.0.0:8008
 bootstrap:
@@ -68,7 +69,8 @@ zookeeper:
   exhibitor:
     hosts: [localhost]
     port: 8181
-""")
+"""
+        self.config = Config()
         self.postgresql = p
         self.dcs = d
         self.api = Mock()

tests/test_patroni.py

@@ -1,5 +1,4 @@
 import etcd
-import os
 import sys
 import time
 import unittest
@@ -34,7 +33,8 @@ class TestPatroni(unittest.TestCase):
         RestApiServer.socket = 0
         with patch.object(etcd.Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
-            self.p = Patroni('postgres0.yml')
+            sys.argv = ['patroni.py', 'postgres0.yml']
+            self.p = Patroni()
 
     @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(side_effect=[None, DCSError('foo'), None]))
     def test_load_dynamic_configuration(self):
@@ -47,7 +47,6 @@ class TestPatroni(unittest.TestCase):
     @patch.object(etcd.Client, 'machines')
     def test_patroni_main(self, mock_machines):
         with patch('subprocess.call', Mock(return_value=1)):
-            _main()
             sys.argv = ['patroni.py', 'postgres0.yml']
 
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
@@ -55,13 +54,6 @@ class TestPatroni(unittest.TestCase):
                 self.assertRaises(SleepException, _main)
             with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())):
                 _main()
-            sys.argv = ['patroni.py']
-            # read the content of the yaml configuration file into the environment variable
-            # in order to test how does patroni handle the configuration passed from the environment.
-            with open('postgres0.yml', 'r') as f:
-                os.environ[Patroni.PATRONI_CONFIG_VARIABLE] = f.read()
-            with patch.object(Patroni, 'run', Mock(side_effect=SleepException())):
-                self.assertRaises(SleepException, _main)
 
     @patch('patroni.config.Config.save_cache', Mock())
     @patch('patroni.config.Config.reload_local_configuration', Mock(return_value=True))