From b7d87f7d07486335428e4469007525816977f655 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 8 Jun 2016 10:15:24 +0200 Subject: [PATCH] Implement possibility to configure Patroni via environment --- README.rst | 8 ++- docs/ENVIRONMENT.rst | 52 +++++++++++++++++ SETTINGS.rst => docs/SETTINGS.rst | 0 patroni/__init__.py | 2 +- patroni/api.py | 23 +------- patroni/config.py | 97 +++++++++++++++++++++++++++++-- patroni/utils.py | 21 +++++++ postgres0.yml | 3 + tests/test_config.py | 30 +++++++++- tests/test_ha.py | 2 + tests/test_patroni.py | 1 - 11 files changed, 209 insertions(+), 30 deletions(-) create mode 100644 docs/ENVIRONMENT.rst rename SETTINGS.rst => docs/SETTINGS.rst (100%) diff --git a/README.rst b/README.rst index 922f1e25..edf0357e 100644 --- a/README.rst +++ b/README.rst @@ -75,7 +75,13 @@ run: YAML Configuration =============== -Go `here `__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml `__. +Go `here `__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml `__. + +========================= +Environment Configuration +========================= + +Go `here `__ for comprehensive information about configuring(overriding) settings via environment variables. =============== Replication Choices diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst new file mode 100644 index 00000000..c8ac4528 --- /dev/null +++ b/docs/ENVIRONMENT.rst @@ -0,0 +1,52 @@ +================================== +Environment Configuration Settings +================================== + +Some of configuration parameters defined in the configuration file is possible to override via Environment variables +This document list all possible environment variables handled by Patroni. +Environment variable always takes precedence on configuration file. + +Global/Universal +---------------- +- **PATRONI\_NAME**: name of the node where the current instance of Patroni is running. Must be unique for the cluster. +- **PATRONI\_NAMESPACE**: path within configuration store where Patroni will keep information about cluster. Default value: "/service" +- **PATRONI\_SCOPE**: cluster name + +Bootstrap configuration +----------------------- +It is possible to define users which will be created right after initializing of a new cluster by defining following environment variables: +- **PATRONI\_\_PASSWORD=''** +- **PATRONI\_\_OPTIONS='list,of,options'** +Example: defining of `PATRONI\_admin\_PASSWORD=admin` `PATRONI\_admin\_OPTIONS='createrole,createdb'` will cause creation of `admin` user which is allowed to create other users and databases + +Consul +------ +- **PATRONI\_CONSUL\_HOST**: the host:port for the Consul endpoint. + +Etcd +---- +- **PATRONI\_ETCD\_HOST**: the host:port for the etcd endpoint. + +PostgreSQL +---------- +- **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node. +- **PATRONI\_POSTGRESQL\_CONNECT\_ADDRESS**: IP address + port through which Postgres is accessible from other nodes and applications. +- **PATRONI\_POSTGRESQL\_DATA\_DIR**: file path to initialize and store Postgres data files. +- **PATRONI\_POSTGRESQL\_PGPASS**: path to `pgpass` file which would be created by Patroni when it is necessary (for example, before executing pg\_basebackup). This locations must be accessible for writing by Patroni. +- **PATRONI\_REPLICATION\_USERNAME**: replication username; user will be created during initialization. Replicas will use this user to access master via streaming replication +- **PATRONI\_REPLICATION\_PASSWORD**: replication password; user will be created during initialization. +- **PATRONI\_SUPERUSER\_USERNAME**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres. Also this user is used by pg_rewind. +- **PATRONI\_SUPERUSER\_PASSWORD**: password for the superuser, set during initialization (initdb). + +REST API +-------- +- **PATRONI\_RESTAPI\_CONNECT\_ADDRESS**: IP address and port through which restapi is accessible. +- **PATRONI\_RESTAPI\_LISTEN**: IP address and port that Patroni will listen to, to provide health-check information for HAProxy. +- **PATRONI\_RESTAPI\_USERNAME**: Basic-auth username to protect dangerous REST API endpoints. +- **PATRONI\_RESTAPI\_PASSWORD**: Basic-auth password to protect dangerous REST API endpoints. +- **PATRONI\_RESTAPI\_CERTFILE**: Specifies a file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL. +- **PATRONI\_RESTAPI\_KEYFILE**: Specifies a file with the secret key in the PEM format. + +ZooKeeper +--------- +- **PATRONI\_ZOOKEEPER\_HOSTS**: comma separated list of ZooKeeper cluster members: 'host1:port1,host2:port2,etc...' diff --git a/SETTINGS.rst b/docs/SETTINGS.rst similarity index 100% rename from SETTINGS.rst rename to docs/SETTINGS.rst diff --git a/patroni/__init__.py b/patroni/__init__.py index d8494c23..84a1018d 100644 --- a/patroni/__init__.py +++ b/patroni/__init__.py @@ -120,7 +120,7 @@ def main(): config_env = False config_file = len(sys.argv) >= 2 and os.path.isfile(sys.argv[1]) and sys.argv[1] if not config_file: - config_env = os.environ.get(Patroni.PATRONI_CONFIG_VARIABLE) + config_env = os.environ.pop(Patroni.PATRONI_CONFIG_VARIABLE, None) if config_env is None: print('Usage: {0} config.yml'.format(sys.argv[0])) print('\tPatroni may also read the configuration from the {} environment variable'. diff --git a/patroni/api.py b/patroni/api.py index c185c755..420067ca 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -9,7 +9,7 @@ import datetime import pytz from patroni.exceptions import PostgresConnectionException -from patroni.utils import deep_compare, Retry, RetryFailedError +from patroni.utils import deep_compare, patch_config, Retry, RetryFailedError from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer from six.moves.socketserver import ThreadingMixIn from threading import Thread @@ -108,25 +108,6 @@ class RestApiHandler(BaseHTTPRequestHandler): def do_GET_config(self): self._write_json_response(200, self.server.patroni.config.dynamic_configuration) - @staticmethod - def _patch_config(config, data): - is_changed = False - for name, value in data.items(): - if value is None: - if config.pop(name, None) is not None: - is_changed = True - elif name in config: - if isinstance(value, dict): - if RestApiHandler._patch_config(config[name], value): - is_changed = True - elif str(config[name]) != str(value): - config[name] = value - is_changed = True - else: - config[name] = value - is_changed = True - return is_changed - def _read_json_content(self): if 'content-length' not in self.headers: return self.send_error(411) @@ -145,7 +126,7 @@ class RestApiHandler(BaseHTTPRequestHandler): if request: cluster = self.server.patroni.ha.dcs.get_cluster() data = cluster.config.data.copy() - if RestApiHandler._patch_config(data, request): + if patch_config(data, request): value = json.dumps(data, separators=(',', ':')) if not self.server.patroni.ha.dcs.set_config_value(value, cluster.config.index): return self.send_error(409) diff --git a/patroni/config.py b/patroni/config.py index 541905a0..f8ec2686 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -4,10 +4,11 @@ import os import tempfile import yaml +from collections import defaultdict from copy import deepcopy from patroni.dcs import ClusterConfig from patroni.postgresql import Postgresql -from patroni.utils import deep_compare +from patroni.utils import deep_compare, patch_config logger = logging.getLogger(__name__) @@ -45,7 +46,11 @@ class Config(object): self._config_file = None if config_env else config_file self._modify_index = -1 self._dynamic_configuration = {} - self._local_configuration = yaml.safe_load(config_env) if config_env else self._load_config_file() + if config_env: + self._local_configuration = yaml.safe_load(config_env) + else: + self.__environment_configuration = self._build_environment_configuration() + self._local_configuration = self._load_config_file() self.__effective_configuration = self._build_effective_configuration(self._dynamic_configuration, self._local_configuration) self._data_dir = self.__effective_configuration['postgresql']['data_dir'] @@ -62,8 +67,11 @@ class Config(object): return deepcopy(self._dynamic_configuration) def _load_config_file(self): + """Loads config.yaml from filesystem and applies some values which were set via ENV""" with open(self._config_file) as f: - return yaml.safe_load(f) + config = yaml.safe_load(f) + patch_config(config, self.__environment_configuration) + return config def _load_cache(self): if os.path.isfile(self._cache_file): @@ -151,10 +159,86 @@ class Config(object): config['postgresql'][name].update(self._process_postgresql_parameters(value)) elif name not in ('connect_address', 'listen', 'data_dir', 'pgpass', 'authentication'): config['postgresql'][name] = deepcopy(value) - elif name in config: + elif name in config: # only variables present in __DEFAULT_CONFIG allowed to be overriden from DCS config[name] = int(value) return config + @staticmethod + def _build_environment_configuration(): + ret = defaultdict(dict) + def _popenv(name): + return os.environ.pop('PATRONI_' + name.upper(), None) + + for param in ('name', 'namespace', 'scope'): + value = _popenv(param) + if value: + ret[param] = value + + def _set_section_values(section, params): + for param in params: + value = _popenv(section + '_' + param) + if value: + ret[section][param] = value + + _set_section_values('restapi', ['listen', 'connect_address', 'certfile', 'keyfile']) + _set_section_values('postgresql', ['listen', 'connect_address', 'data_dir', 'pgpass']) + + def _get_auth(name): + ret = {} + for param in ('username', 'password'): + value = _popenv(name + '_' + param) + if value: + ret[param] = value + return len(ret) == 2 and ret or None + + restapi_auth = _get_auth('restapi') + if restapi_auth: + ret['restapi']['authentication'] = restapi_auth + + authentication = {} + for user_type in ('replication', 'superuser'): + entry = _get_auth(user_type) + if entry: + authentication[user_type] = entry + + if authentication: + ret['postgresql']['authentication'] = authentication + + users = {} + + def _parse_list(value): + if not (value.strip().startswith('-') or '[' in value): + value = '[{0}]'.format(value) + try: + return yaml.safe_load(value) + except Exception: + return None + + for param in list(os.environ.keys()): + if param.startswith('PATRONI_'): + name, suffix = (param[8:].rsplit('_', 1) + [''])[:2] + if name and suffix: + # PATRONI_(ETCD|CONSUL|ZOOKEEPER|...)_HOSTS? + if suffix in ('HOST', 'HOSTS') and '_' not in name: + value = os.environ.pop(param) + value = value if suffix == 'HOST' else value and _parse_list(value) + if value: + ret[name.lower()][suffix.lower()] = value + # PATRONI__PASSWORD=, PATRONI__OPTIONS= + # CREATE USER "" WITH PASSWORD '' + elif suffix == 'PASSWORD': + password = os.environ.pop(param) + if password: + users[name] = {'password': password} + options = os.environ.pop(param[:-9] + '_OPTIONS', None) + options = options and _parse_list(options) + if options: + users[name]['options'] = options + if users: + ret['bootstrap']['users'] = users + + return ret + def _build_effective_configuration(self, dynamic_configuration, local_configuration): config = self._safe_copy_dynamic_configuration(dynamic_configuration) for name, value in local_configuration.items(): @@ -167,6 +251,11 @@ class Config(object): elif name not in config: config[name] = deepcopy(value) if value else {} + if 'authentication' in config['restapi']: + restapi = config['restapi'] + auth = restapi['authentication'] + restapi['auth'] = '{0}:{1}'.format(auth['username'], auth['password']) + pg_config = config['postgresql'] # special treatment for old config diff --git a/patroni/utils.py b/patroni/utils.py index f85ec56f..fdeed596 100644 --- a/patroni/utils.py +++ b/patroni/utils.py @@ -60,6 +60,27 @@ def deep_compare(obj1, obj2): return True +def patch_config(config, data): + """recursively 'patch' `config` with `data` + :returns: `!True` if the `config` was changed""" + is_changed = False + for name, value in data.items(): + if value is None: + if config.pop(name, None) is not None: + is_changed = True + elif name in config: + if isinstance(value, dict): + if patch_config(config[name], value): + is_changed = True + elif str(config[name]) != str(value): + config[name] = value + is_changed = True + else: + config[name] = value + is_changed = True + return is_changed + + def parse_bool(value): """ >>> parse_bool(1) diff --git a/postgres0.yml b/postgres0.yml index e09cb4f0..c05788c4 100644 --- a/postgres0.yml +++ b/postgres0.yml @@ -4,6 +4,9 @@ name: postgresql0 restapi: listen: 127.0.0.1:8008 +# authentication: +# username: username +# password: password connect_address: 127.0.0.1:8008 etcd: diff --git a/tests/test_config.py b/tests/test_config.py index 6987fa67..d85d4f3a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,3 +1,4 @@ +import os import unittest from mock import MagicMock, Mock, patch @@ -11,15 +12,40 @@ class TestConfig(unittest.TestCase): @patch('json.load', Mock(side_effect=Exception)) @patch.object(builtins, 'open', MagicMock()) def setUp(self): - self.config = Config(config_env='postgresql: {data_dir: foo}') + self.config = Config(config_env='restapi: {}\npostgresql: {data_dir: foo}') @patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)) def test_set_dynamic_configuration(self): self.assertIsNone(self.config.set_dynamic_configuration({'foo': 'bar'})) def test_reload_local_configuration(self): + os.environ.update({ + 'PATRONI_NAME': 'postgres0', + 'PATRONI_NAMESPACE': '/patroni/', + 'PATRONI_SCOPE': 'batman2', + 'PATRONI_RESTAPI_USERNAME': 'username', + 'PATRONI_RESTAPI_PASSWORD': 'password', + 'PATRONI_RESTAPI_LISTEN': '0.0.0.0:8008', + 'PATRONI_RESTAPI_CONNECT_ADDRESS': '127.0.0.1:8008', + 'PATRONI_RESTAPI_CERTFILE': '/certfile', + 'PATRONI_RESTAPI_KEYFILE': '/keyfile', + 'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432', + 'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432', + 'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0', + 'PATRONI_POSTGRESQL_PGPASS': '/tmp/pgpass0', + 'PATRONI_ETCD_HOST': '127.0.0.1:2379', + 'PATRONI_CONSUL_HOST': '127.0.0.1:8500', + 'PATRONI_ZOOKEEPER_HOSTS': 'host1,host2', + 'PATRONI_foo_HOSTS': '[host1,host2', # Exception in parse_list + 'PATRONI_SUPERUSER_USERNAME': 'postgres', + 'PATRONI_SUPERUSER_PASSWORD': 'zalando', + 'PATRONI_REPLICATION_USERNAME': 'replicator', + 'PATRONI_REPLICATION_PASSWORD': 'rep-pass', + 'PATRONI_admin_PASSWORD': 'admin', + 'PATRONI_admin_OPTIONS': 'createrole,createdb' + }) config = Config(config_file='postgres0.yml') - with patch.object(Config, '_load_config_file', Mock(return_value={})): + with patch.object(Config, '_load_config_file', Mock(return_value={'restapi': {}})): with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)): self.assertRaises(Exception, config.reload_local_configuration, True) self.assertTrue(config.reload_local_configuration(True)) diff --git a/tests/test_ha.py b/tests/test_ha.py index ee90eae0..911401da 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -50,6 +50,8 @@ class MockPatroni(object): def __init__(self, p, d): self.config = Config(config_env=""" +restapi: + listen: 0.0.0.0:8008 bootstrap: users: replicator: diff --git a/tests/test_patroni.py b/tests/test_patroni.py index 1e53e27b..a16b611d 100644 --- a/tests/test_patroni.py +++ b/tests/test_patroni.py @@ -61,7 +61,6 @@ class TestPatroni(unittest.TestCase): os.environ[Patroni.PATRONI_CONFIG_VARIABLE] = f.read() with patch.object(Patroni, 'run', Mock(side_effect=SleepException())): self.assertRaises(SleepException, _main) - del os.environ[Patroni.PATRONI_CONFIG_VARIABLE] @patch('patroni.config.Config.save_cache', Mock()) @patch('patroni.config.Config.reload_local_configuration', Mock(return_value=True))