From 49efb371f9b46e0552d32c999dd03b80de27f678 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 9 Jun 2016 14:44:29 +0200 Subject: [PATCH] Make it possible to work without config.yml Most of the basic configuration could be done via ENV --- patroni/__init__.py | 20 +++----------------- patroni/config.py | 30 ++++++++++++++++++++++-------- tests/test_config.py | 11 +++++++++-- tests/test_ha.py | 10 ++++++---- tests/test_patroni.py | 12 ++---------- 5 files changed, 42 insertions(+), 41 deletions(-) diff --git a/patroni/__init__.py b/patroni/__init__.py index 84a1018d..c48eab40 100644 --- a/patroni/__init__.py +++ b/patroni/__init__.py @@ -1,7 +1,5 @@ import logging -import os import signal -import sys import time from patroni.api import RestApiServer @@ -17,11 +15,10 @@ logger = logging.getLogger(__name__) class Patroni(object): - PATRONI_CONFIG_VARIABLE = 'PATRONI_CONFIGURATION' - def __init__(self, config_file=None, config_env=None): + def __init__(self): self.version = __version__ - self.config = Config(config_file=config_file, config_env=config_env) + self.config = Config() self.dcs = get_dcs(self.config) self.load_dynamic_configuration() @@ -116,18 +113,7 @@ def main(): logging.getLogger('requests').setLevel(logging.WARNING) setup_signal_handlers() - # Patroni reads the configuration from the command-line argument if it exists, and from the environment otherwise. - config_env = False - config_file = len(sys.argv) >= 2 and os.path.isfile(sys.argv[1]) and sys.argv[1] - if not config_file: - config_env = os.environ.pop(Patroni.PATRONI_CONFIG_VARIABLE, None) - if config_env is None: - print('Usage: {0} config.yml'.format(sys.argv[0])) - print('\tPatroni may also read the configuration from the {} environment variable'. - format(Patroni.PATRONI_CONFIG_VARIABLE)) - return - - patroni = Patroni(config_file, config_env) + patroni = Patroni() try: patroni.run() except KeyboardInterrupt: diff --git a/patroni/config.py b/patroni/config.py index 07fd508f..d982842a 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -1,6 +1,7 @@ import json import logging import os +import sys import tempfile import yaml @@ -33,6 +34,9 @@ class Config(object): to work with it as with the old `config` object. """ + PATRONI_ENV_PREFIX = 'PATRONI_' + PATRONI_CONFIG_VARIABLE = PATRONI_ENV_PREFIX + 'CONFIGURATION' + __CACHE_FILENAME = 'patroni.dynamic.json' __DEFAULT_CONFIG = { 'ttl': 30, 'loop_wait': 10, 'retry_timeout': 10, @@ -42,15 +46,25 @@ class Config(object): } } - def __init__(self, config_file=None, config_env=None): - self._config_file = None if config_env else config_file + def __init__(self): self._modify_index = -1 self._dynamic_configuration = {} - if config_env: - self._local_configuration = yaml.safe_load(config_env) - else: - self.__environment_configuration = self._build_environment_configuration() + + self.__environment_configuration = self._build_environment_configuration() + + # Patroni reads the configuration from the command-line argument if it exists, otherwise from the environment + self._config_file = len(sys.argv) >= 2 and os.path.isfile(sys.argv[1]) and sys.argv[1] + if self._config_file: self._local_configuration = self._load_config_file() + else: + config_env = os.environ.pop(self.PATRONI_CONFIG_VARIABLE, None) + self._local_configuration = config_env and yaml.safe_load(config_env) or self.__environment_configuration + if not self._local_configuration: + print('Usage: {0} config.yml'.format(sys.argv[0])) + print('\tPatroni may also read the configuration from the {0} environment variable'. + format(self.PATRONI_CONFIG_VARIABLE)) + exit(1) + self.__effective_configuration = self._build_effective_configuration(self._dynamic_configuration, self._local_configuration) self._data_dir = self.__effective_configuration['postgresql']['data_dir'] @@ -168,7 +182,7 @@ class Config(object): ret = defaultdict(dict) def _popenv(name): - return os.environ.pop('PATRONI_' + name.upper(), None) + return os.environ.pop(Config.PATRONI_ENV_PREFIX + name.upper(), None) for param in ('name', 'namespace', 'scope'): value = _popenv(param) @@ -216,7 +230,7 @@ class Config(object): return None for param in list(os.environ.keys()): - if param.startswith('PATRONI_'): + if param.startswith(Config.PATRONI_ENV_PREFIX): name, suffix = (param[8:].rsplit('_', 1) + [''])[:2] if name and suffix: # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT) diff --git a/tests/test_config.py b/tests/test_config.py index e6d212d6..a8a5a6c4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,6 @@ import os import unittest +import sys from mock import MagicMock, Mock, patch from patroni.config import Config @@ -12,7 +13,12 @@ class TestConfig(unittest.TestCase): @patch('json.load', Mock(side_effect=Exception)) @patch.object(builtins, 'open', MagicMock()) def setUp(self): - self.config = Config(config_env='restapi: {}\npostgresql: {data_dir: foo}') + sys.argv = ['patroni.py'] + os.environ[Config.PATRONI_CONFIG_VARIABLE] = 'restapi: {}\npostgresql: {data_dir: foo}' + self.config = Config() + + def test_no_config(self): + self.assertRaises(SystemExit, Config) @patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)) def test_set_dynamic_configuration(self): @@ -46,7 +52,8 @@ class TestConfig(unittest.TestCase): 'PATRONI_admin_PASSWORD': 'admin', 'PATRONI_admin_OPTIONS': 'createrole,createdb' }) - config = Config(config_file='postgres0.yml') + sys.argv = ['patroni.py', 'postgres0.yml'] + config = Config() with patch.object(Config, '_load_config_file', Mock(return_value={'restapi': {}})): with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)): self.assertRaises(Exception, config.reload_local_configuration, True) diff --git a/tests/test_ha.py b/tests/test_ha.py index 59be45cd..4f691212 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -1,7 +1,8 @@ -import etcd -import unittest import datetime +import etcd +import os import pytz +import unittest from mock import Mock, MagicMock, patch from patroni.config import Config @@ -49,7 +50,7 @@ def get_cluster_initialized_with_only_leader(failover=None): class MockPatroni(object): def __init__(self, p, d): - self.config = Config(config_env=""" + os.environ[Config.PATRONI_CONFIG_VARIABLE] = """ restapi: listen: 0.0.0.0:8008 bootstrap: @@ -68,7 +69,8 @@ zookeeper: exhibitor: hosts: [localhost] port: 8181 -""") +""" + self.config = Config() self.postgresql = p self.dcs = d self.api = Mock() diff --git a/tests/test_patroni.py b/tests/test_patroni.py index b685f2ae..f9246cac 100644 --- a/tests/test_patroni.py +++ b/tests/test_patroni.py @@ -1,5 +1,4 @@ import etcd -import os import sys import time import unittest @@ -34,7 +33,8 @@ class TestPatroni(unittest.TestCase): RestApiServer.socket = 0 with patch.object(etcd.Client, 'machines') as mock_machines: mock_machines.__get__ = Mock(return_value=['http://remotehost:2379']) - self.p = Patroni('postgres0.yml') + sys.argv = ['patroni.py', 'postgres0.yml'] + self.p = Patroni() @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(side_effect=[None, DCSError('foo'), None])) def test_load_dynamic_configuration(self): @@ -47,7 +47,6 @@ class TestPatroni(unittest.TestCase): @patch.object(etcd.Client, 'machines') def test_patroni_main(self, mock_machines): with patch('subprocess.call', Mock(return_value=1)): - _main() sys.argv = ['patroni.py', 'postgres0.yml'] mock_machines.__get__ = Mock(return_value=['http://remotehost:2379']) @@ -55,13 +54,6 @@ class TestPatroni(unittest.TestCase): self.assertRaises(SleepException, _main) with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())): _main() - sys.argv = ['patroni.py'] - # read the content of the yaml configuration file into the environment variable - # in order to test how does patroni handle the configuration passed from the environment. - with open('postgres0.yml', 'r') as f: - os.environ[Patroni.PATRONI_CONFIG_VARIABLE] = f.read() - with patch.object(Patroni, 'run', Mock(side_effect=SleepException())): - self.assertRaises(SleepException, _main) @patch('patroni.config.Config.save_cache', Mock()) @patch('patroni.config.Config.reload_local_configuration', Mock(return_value=True))