From b7d87f7d07486335428e4469007525816977f655 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 8 Jun 2016 10:15:24 +0200 Subject: [PATCH 1/8] Implement possibility to configure Patroni via environment --- README.rst | 8 ++- docs/ENVIRONMENT.rst | 52 +++++++++++++++++ SETTINGS.rst => docs/SETTINGS.rst | 0 patroni/__init__.py | 2 +- patroni/api.py | 23 +------- patroni/config.py | 97 +++++++++++++++++++++++++++++-- patroni/utils.py | 21 +++++++ postgres0.yml | 3 + tests/test_config.py | 30 +++++++++- tests/test_ha.py | 2 + tests/test_patroni.py | 1 - 11 files changed, 209 insertions(+), 30 deletions(-) create mode 100644 docs/ENVIRONMENT.rst rename SETTINGS.rst => docs/SETTINGS.rst (100%) diff --git a/README.rst b/README.rst index 922f1e25..edf0357e 100644 --- a/README.rst +++ b/README.rst @@ -75,7 +75,13 @@ run: YAML Configuration =============== -Go `here `__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml `__. +Go `here `__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml `__. + +========================= +Environment Configuration +========================= + +Go `here `__ for comprehensive information about configuring(overriding) settings via environment variables. =============== Replication Choices diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst new file mode 100644 index 00000000..c8ac4528 --- /dev/null +++ b/docs/ENVIRONMENT.rst @@ -0,0 +1,52 @@ +================================== +Environment Configuration Settings +================================== + +Some of configuration parameters defined in the configuration file is possible to override via Environment variables +This document list all possible environment variables handled by Patroni. +Environment variable always takes precedence on configuration file. + +Global/Universal +---------------- +- **PATRONI\_NAME**: name of the node where the current instance of Patroni is running. Must be unique for the cluster. +- **PATRONI\_NAMESPACE**: path within configuration store where Patroni will keep information about cluster. Default value: "/service" +- **PATRONI\_SCOPE**: cluster name + +Bootstrap configuration +----------------------- +It is possible to define users which will be created right after initializing of a new cluster by defining following environment variables: +- **PATRONI\_\_PASSWORD=''** +- **PATRONI\_\_OPTIONS='list,of,options'** +Example: defining of `PATRONI\_admin\_PASSWORD=admin` `PATRONI\_admin\_OPTIONS='createrole,createdb'` will cause creation of `admin` user which is allowed to create other users and databases + +Consul +------ +- **PATRONI\_CONSUL\_HOST**: the host:port for the Consul endpoint. + +Etcd +---- +- **PATRONI\_ETCD\_HOST**: the host:port for the etcd endpoint. + +PostgreSQL +---------- +- **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node. +- **PATRONI\_POSTGRESQL\_CONNECT\_ADDRESS**: IP address + port through which Postgres is accessible from other nodes and applications. +- **PATRONI\_POSTGRESQL\_DATA\_DIR**: file path to initialize and store Postgres data files. +- **PATRONI\_POSTGRESQL\_PGPASS**: path to `pgpass` file which would be created by Patroni when it is necessary (for example, before executing pg\_basebackup). This locations must be accessible for writing by Patroni. +- **PATRONI\_REPLICATION\_USERNAME**: replication username; user will be created during initialization. Replicas will use this user to access master via streaming replication +- **PATRONI\_REPLICATION\_PASSWORD**: replication password; user will be created during initialization. +- **PATRONI\_SUPERUSER\_USERNAME**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres. Also this user is used by pg_rewind. +- **PATRONI\_SUPERUSER\_PASSWORD**: password for the superuser, set during initialization (initdb). + +REST API +-------- +- **PATRONI\_RESTAPI\_CONNECT\_ADDRESS**: IP address and port through which restapi is accessible. +- **PATRONI\_RESTAPI\_LISTEN**: IP address and port that Patroni will listen to, to provide health-check information for HAProxy. +- **PATRONI\_RESTAPI\_USERNAME**: Basic-auth username to protect dangerous REST API endpoints. +- **PATRONI\_RESTAPI\_PASSWORD**: Basic-auth password to protect dangerous REST API endpoints. +- **PATRONI\_RESTAPI\_CERTFILE**: Specifies a file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL. +- **PATRONI\_RESTAPI\_KEYFILE**: Specifies a file with the secret key in the PEM format. + +ZooKeeper +--------- +- **PATRONI\_ZOOKEEPER\_HOSTS**: comma separated list of ZooKeeper cluster members: 'host1:port1,host2:port2,etc...' diff --git a/SETTINGS.rst b/docs/SETTINGS.rst similarity index 100% rename from SETTINGS.rst rename to docs/SETTINGS.rst diff --git a/patroni/__init__.py b/patroni/__init__.py index d8494c23..84a1018d 100644 --- a/patroni/__init__.py +++ b/patroni/__init__.py @@ -120,7 +120,7 @@ def main(): config_env = False config_file = len(sys.argv) >= 2 and os.path.isfile(sys.argv[1]) and sys.argv[1] if not config_file: - config_env = os.environ.get(Patroni.PATRONI_CONFIG_VARIABLE) + config_env = os.environ.pop(Patroni.PATRONI_CONFIG_VARIABLE, None) if config_env is None: print('Usage: {0} config.yml'.format(sys.argv[0])) print('\tPatroni may also read the configuration from the {} environment variable'. diff --git a/patroni/api.py b/patroni/api.py index c185c755..420067ca 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -9,7 +9,7 @@ import datetime import pytz from patroni.exceptions import PostgresConnectionException -from patroni.utils import deep_compare, Retry, RetryFailedError +from patroni.utils import deep_compare, patch_config, Retry, RetryFailedError from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer from six.moves.socketserver import ThreadingMixIn from threading import Thread @@ -108,25 +108,6 @@ class RestApiHandler(BaseHTTPRequestHandler): def do_GET_config(self): self._write_json_response(200, self.server.patroni.config.dynamic_configuration) - @staticmethod - def _patch_config(config, data): - is_changed = False - for name, value in data.items(): - if value is None: - if config.pop(name, None) is not None: - is_changed = True - elif name in config: - if isinstance(value, dict): - if RestApiHandler._patch_config(config[name], value): - is_changed = True - elif str(config[name]) != str(value): - config[name] = value - is_changed = True - else: - config[name] = value - is_changed = True - return is_changed - def _read_json_content(self): if 'content-length' not in self.headers: return self.send_error(411) @@ -145,7 +126,7 @@ class RestApiHandler(BaseHTTPRequestHandler): if request: cluster = self.server.patroni.ha.dcs.get_cluster() data = cluster.config.data.copy() - if RestApiHandler._patch_config(data, request): + if patch_config(data, request): value = json.dumps(data, separators=(',', ':')) if not self.server.patroni.ha.dcs.set_config_value(value, cluster.config.index): return self.send_error(409) diff --git a/patroni/config.py b/patroni/config.py index 541905a0..f8ec2686 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -4,10 +4,11 @@ import os import tempfile import yaml +from collections import defaultdict from copy import deepcopy from patroni.dcs import ClusterConfig from patroni.postgresql import Postgresql -from patroni.utils import deep_compare +from patroni.utils import deep_compare, patch_config logger = logging.getLogger(__name__) @@ -45,7 +46,11 @@ class Config(object): self._config_file = None if config_env else config_file self._modify_index = -1 self._dynamic_configuration = {} - self._local_configuration = yaml.safe_load(config_env) if config_env else self._load_config_file() + if config_env: + self._local_configuration = yaml.safe_load(config_env) + else: + self.__environment_configuration = self._build_environment_configuration() + self._local_configuration = self._load_config_file() self.__effective_configuration = self._build_effective_configuration(self._dynamic_configuration, self._local_configuration) self._data_dir = self.__effective_configuration['postgresql']['data_dir'] @@ -62,8 +67,11 @@ class Config(object): return deepcopy(self._dynamic_configuration) def _load_config_file(self): + """Loads config.yaml from filesystem and applies some values which were set via ENV""" with open(self._config_file) as f: - return yaml.safe_load(f) + config = yaml.safe_load(f) + patch_config(config, self.__environment_configuration) + return config def _load_cache(self): if os.path.isfile(self._cache_file): @@ -151,10 +159,86 @@ class Config(object): config['postgresql'][name].update(self._process_postgresql_parameters(value)) elif name not in ('connect_address', 'listen', 'data_dir', 'pgpass', 'authentication'): config['postgresql'][name] = deepcopy(value) - elif name in config: + elif name in config: # only variables present in __DEFAULT_CONFIG allowed to be overriden from DCS config[name] = int(value) return config + @staticmethod + def _build_environment_configuration(): + ret = defaultdict(dict) + def _popenv(name): + return os.environ.pop('PATRONI_' + name.upper(), None) + + for param in ('name', 'namespace', 'scope'): + value = _popenv(param) + if value: + ret[param] = value + + def _set_section_values(section, params): + for param in params: + value = _popenv(section + '_' + param) + if value: + ret[section][param] = value + + _set_section_values('restapi', ['listen', 'connect_address', 'certfile', 'keyfile']) + _set_section_values('postgresql', ['listen', 'connect_address', 'data_dir', 'pgpass']) + + def _get_auth(name): + ret = {} + for param in ('username', 'password'): + value = _popenv(name + '_' + param) + if value: + ret[param] = value + return len(ret) == 2 and ret or None + + restapi_auth = _get_auth('restapi') + if restapi_auth: + ret['restapi']['authentication'] = restapi_auth + + authentication = {} + for user_type in ('replication', 'superuser'): + entry = _get_auth(user_type) + if entry: + authentication[user_type] = entry + + if authentication: + ret['postgresql']['authentication'] = authentication + + users = {} + + def _parse_list(value): + if not (value.strip().startswith('-') or '[' in value): + value = '[{0}]'.format(value) + try: + return yaml.safe_load(value) + except Exception: + return None + + for param in list(os.environ.keys()): + if param.startswith('PATRONI_'): + name, suffix = (param[8:].rsplit('_', 1) + [''])[:2] + if name and suffix: + # PATRONI_(ETCD|CONSUL|ZOOKEEPER|...)_HOSTS? + if suffix in ('HOST', 'HOSTS') and '_' not in name: + value = os.environ.pop(param) + value = value if suffix == 'HOST' else value and _parse_list(value) + if value: + ret[name.lower()][suffix.lower()] = value + # PATRONI__PASSWORD=, PATRONI__OPTIONS= + # CREATE USER "" WITH PASSWORD '' + elif suffix == 'PASSWORD': + password = os.environ.pop(param) + if password: + users[name] = {'password': password} + options = os.environ.pop(param[:-9] + '_OPTIONS', None) + options = options and _parse_list(options) + if options: + users[name]['options'] = options + if users: + ret['bootstrap']['users'] = users + + return ret + def _build_effective_configuration(self, dynamic_configuration, local_configuration): config = self._safe_copy_dynamic_configuration(dynamic_configuration) for name, value in local_configuration.items(): @@ -167,6 +251,11 @@ class Config(object): elif name not in config: config[name] = deepcopy(value) if value else {} + if 'authentication' in config['restapi']: + restapi = config['restapi'] + auth = restapi['authentication'] + restapi['auth'] = '{0}:{1}'.format(auth['username'], auth['password']) + pg_config = config['postgresql'] # special treatment for old config diff --git a/patroni/utils.py b/patroni/utils.py index f85ec56f..fdeed596 100644 --- a/patroni/utils.py +++ b/patroni/utils.py @@ -60,6 +60,27 @@ def deep_compare(obj1, obj2): return True +def patch_config(config, data): + """recursively 'patch' `config` with `data` + :returns: `!True` if the `config` was changed""" + is_changed = False + for name, value in data.items(): + if value is None: + if config.pop(name, None) is not None: + is_changed = True + elif name in config: + if isinstance(value, dict): + if patch_config(config[name], value): + is_changed = True + elif str(config[name]) != str(value): + config[name] = value + is_changed = True + else: + config[name] = value + is_changed = True + return is_changed + + def parse_bool(value): """ >>> parse_bool(1) diff --git a/postgres0.yml b/postgres0.yml index e09cb4f0..c05788c4 100644 --- a/postgres0.yml +++ b/postgres0.yml @@ -4,6 +4,9 @@ name: postgresql0 restapi: listen: 127.0.0.1:8008 +# authentication: +# username: username +# password: password connect_address: 127.0.0.1:8008 etcd: diff --git a/tests/test_config.py b/tests/test_config.py index 6987fa67..d85d4f3a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,3 +1,4 @@ +import os import unittest from mock import MagicMock, Mock, patch @@ -11,15 +12,40 @@ class TestConfig(unittest.TestCase): @patch('json.load', Mock(side_effect=Exception)) @patch.object(builtins, 'open', MagicMock()) def setUp(self): - self.config = Config(config_env='postgresql: {data_dir: foo}') + self.config = Config(config_env='restapi: {}\npostgresql: {data_dir: foo}') @patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)) def test_set_dynamic_configuration(self): self.assertIsNone(self.config.set_dynamic_configuration({'foo': 'bar'})) def test_reload_local_configuration(self): + os.environ.update({ + 'PATRONI_NAME': 'postgres0', + 'PATRONI_NAMESPACE': '/patroni/', + 'PATRONI_SCOPE': 'batman2', + 'PATRONI_RESTAPI_USERNAME': 'username', + 'PATRONI_RESTAPI_PASSWORD': 'password', + 'PATRONI_RESTAPI_LISTEN': '0.0.0.0:8008', + 'PATRONI_RESTAPI_CONNECT_ADDRESS': '127.0.0.1:8008', + 'PATRONI_RESTAPI_CERTFILE': '/certfile', + 'PATRONI_RESTAPI_KEYFILE': '/keyfile', + 'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432', + 'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432', + 'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0', + 'PATRONI_POSTGRESQL_PGPASS': '/tmp/pgpass0', + 'PATRONI_ETCD_HOST': '127.0.0.1:2379', + 'PATRONI_CONSUL_HOST': '127.0.0.1:8500', + 'PATRONI_ZOOKEEPER_HOSTS': 'host1,host2', + 'PATRONI_foo_HOSTS': '[host1,host2', # Exception in parse_list + 'PATRONI_SUPERUSER_USERNAME': 'postgres', + 'PATRONI_SUPERUSER_PASSWORD': 'zalando', + 'PATRONI_REPLICATION_USERNAME': 'replicator', + 'PATRONI_REPLICATION_PASSWORD': 'rep-pass', + 'PATRONI_admin_PASSWORD': 'admin', + 'PATRONI_admin_OPTIONS': 'createrole,createdb' + }) config = Config(config_file='postgres0.yml') - with patch.object(Config, '_load_config_file', Mock(return_value={})): + with patch.object(Config, '_load_config_file', Mock(return_value={'restapi': {}})): with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)): self.assertRaises(Exception, config.reload_local_configuration, True) self.assertTrue(config.reload_local_configuration(True)) diff --git a/tests/test_ha.py b/tests/test_ha.py index ee90eae0..911401da 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -50,6 +50,8 @@ class MockPatroni(object): def __init__(self, p, d): self.config = Config(config_env=""" +restapi: + listen: 0.0.0.0:8008 bootstrap: users: replicator: diff --git a/tests/test_patroni.py b/tests/test_patroni.py index 1e53e27b..a16b611d 100644 --- a/tests/test_patroni.py +++ b/tests/test_patroni.py @@ -61,7 +61,6 @@ class TestPatroni(unittest.TestCase): os.environ[Patroni.PATRONI_CONFIG_VARIABLE] = f.read() with patch.object(Patroni, 'run', Mock(side_effect=SleepException())): self.assertRaises(SleepException, _main) - del os.environ[Patroni.PATRONI_CONFIG_VARIABLE] @patch('patroni.config.Config.save_cache', Mock()) @patch('patroni.config.Config.reload_local_configuration', Mock(return_value=True)) From b65dc9a82715ec08d9f5085c0808a3afc18ad98c Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 8 Jun 2016 10:29:37 +0200 Subject: [PATCH 2/8] Update ENVIRONMENT.rst --- docs/ENVIRONMENT.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst index c8ac4528..9fc3e398 100644 --- a/docs/ENVIRONMENT.rst +++ b/docs/ENVIRONMENT.rst @@ -15,9 +15,11 @@ Global/Universal Bootstrap configuration ----------------------- It is possible to define users which will be created right after initializing of a new cluster by defining following environment variables: + - **PATRONI\_\_PASSWORD=''** - **PATRONI\_\_OPTIONS='list,of,options'** -Example: defining of `PATRONI\_admin\_PASSWORD=admin` `PATRONI\_admin\_OPTIONS='createrole,createdb'` will cause creation of `admin` user which is allowed to create other users and databases + +Example: defining of ``PATRONI_admin_PASSWORD=strongpasswd`` and ``PATRONI_admin_OPTIONS='createrole,createdb'`` will cause creation of user **admin** with the password **strongpasswd**, which is allowed to create other users and databases. Consul ------ From f2fc68acde7130c7d55f398ad13459ce14a2e6e7 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 8 Jun 2016 10:31:59 +0200 Subject: [PATCH 3/8] Fix pep8 formatting --- patroni/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/patroni/config.py b/patroni/config.py index f8ec2686..31e849c6 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -166,6 +166,7 @@ class Config(object): @staticmethod def _build_environment_configuration(): ret = defaultdict(dict) + def _popenv(name): return os.environ.pop('PATRONI_' + name.upper(), None) From 23c5040ce561dc8a5103fe9d58dfc3ed7004f3a1 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 8 Jun 2016 12:35:53 +0200 Subject: [PATCH 4/8] Update documentation --- docs/ENVIRONMENT.rst | 29 ++++++++++++++--------------- docs/SETTINGS.rst | 22 +++++++++++++--------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst index 9fc3e398..ef29f8c6 100644 --- a/docs/ENVIRONMENT.rst +++ b/docs/ENVIRONMENT.rst @@ -2,24 +2,23 @@ Environment Configuration Settings ================================== -Some of configuration parameters defined in the configuration file is possible to override via Environment variables -This document list all possible environment variables handled by Patroni. -Environment variable always takes precedence on configuration file. +It is possible to override some of the configuration parameters defined in the Patroni configuration file using the system environment variables. This document lists all environment variables handled by Patroni. The values set via those variables always take precedence over the ones set in the Patroni configuration file. Global/Universal ---------------- +- **PATRONI\_CONFIGURATION**: it is possible to set the entire configuration for the Patroni via ``PATRONI_CONFIGURATION`` environment variable. In this case any other environment variables will not be considered! - **PATRONI\_NAME**: name of the node where the current instance of Patroni is running. Must be unique for the cluster. -- **PATRONI\_NAMESPACE**: path within configuration store where Patroni will keep information about cluster. Default value: "/service" +- **PATRONI\_NAMESPACE**: path within the configuration store where Patroni will keep information about the cluster. Default value: "/service" - **PATRONI\_SCOPE**: cluster name Bootstrap configuration ----------------------- -It is possible to define users which will be created right after initializing of a new cluster by defining following environment variables: +It is possible to create new database users right after the successful initialization of a new cluster. This process is defined by the following variables: - **PATRONI\_\_PASSWORD=''** - **PATRONI\_\_OPTIONS='list,of,options'** -Example: defining of ``PATRONI_admin_PASSWORD=strongpasswd`` and ``PATRONI_admin_OPTIONS='createrole,createdb'`` will cause creation of user **admin** with the password **strongpasswd**, which is allowed to create other users and databases. +Example: defining ``PATRONI_admin_PASSWORD=strongpasswd`` and ``PATRONI_admin_OPTIONS='createrole,createdb'`` will cause creation of the user **admin** with the password **strongpasswd** that is allowed to create other users and databases. Consul ------ @@ -33,21 +32,21 @@ PostgreSQL ---------- - **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node. - **PATRONI\_POSTGRESQL\_CONNECT\_ADDRESS**: IP address + port through which Postgres is accessible from other nodes and applications. -- **PATRONI\_POSTGRESQL\_DATA\_DIR**: file path to initialize and store Postgres data files. -- **PATRONI\_POSTGRESQL\_PGPASS**: path to `pgpass` file which would be created by Patroni when it is necessary (for example, before executing pg\_basebackup). This locations must be accessible for writing by Patroni. -- **PATRONI\_REPLICATION\_USERNAME**: replication username; user will be created during initialization. Replicas will use this user to access master via streaming replication -- **PATRONI\_REPLICATION\_PASSWORD**: replication password; user will be created during initialization. +- **PATRONI\_POSTGRESQL\_DATA\_DIR**: The location of the Postgres data directory, either existing or to be initialized by Patroni. +- **PATRONI\_POSTGRESQL\_PGPASS**: path to the [.pgpass password file](https://www.postgresql.org/docs/current/static/libpq-pgpass.html). Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni. +- **PATRONI\_REPLICATION\_USERNAME**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication +- **PATRONI\_REPLICATION\_PASSWORD**: replication password; the user will be created during initialization. - **PATRONI\_SUPERUSER\_USERNAME**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres. Also this user is used by pg_rewind. - **PATRONI\_SUPERUSER\_PASSWORD**: password for the superuser, set during initialization (initdb). REST API -------- -- **PATRONI\_RESTAPI\_CONNECT\_ADDRESS**: IP address and port through which restapi is accessible. +- **PATRONI\_RESTAPI\_CONNECT\_ADDRESS**: IP address and port to access the REST API. - **PATRONI\_RESTAPI\_LISTEN**: IP address and port that Patroni will listen to, to provide health-check information for HAProxy. -- **PATRONI\_RESTAPI\_USERNAME**: Basic-auth username to protect dangerous REST API endpoints. -- **PATRONI\_RESTAPI\_PASSWORD**: Basic-auth password to protect dangerous REST API endpoints. -- **PATRONI\_RESTAPI\_CERTFILE**: Specifies a file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL. -- **PATRONI\_RESTAPI\_KEYFILE**: Specifies a file with the secret key in the PEM format. +- **PATRONI\_RESTAPI\_USERNAME**: Basic-auth username to protect unsafe REST API endpoints. +- **PATRONI\_RESTAPI\_PASSWORD**: Basic-auth password to protect unsafe REST API endpoints. +- **PATRONI\_RESTAPI\_CERTFILE**: Specifies the file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL. +- **PATRONI\_RESTAPI\_KEYFILE**: Specifies the file with the secret key in the PEM format. ZooKeeper --------- diff --git a/docs/SETTINGS.rst b/docs/SETTINGS.rst index 6b0d4153..8524fbe8 100644 --- a/docs/SETTINGS.rst +++ b/docs/SETTINGS.rst @@ -5,7 +5,7 @@ YAML Configuration Settings Global/Universal ---------------- - **name**: the name of the host. Must be unique for the cluster. -- **namespace**: path within configuration store where Patroni will keep information about cluster. Default value: "/service" +- **namespace**: path within the configuration store where Patroni will keep information about the cluster. Default value: "/service" - **scope**: cluster name Bootstrap configuration @@ -48,8 +48,8 @@ PostgreSQL - **username**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres. - **password**: password for the superuser, set during initialization (initdb). - **replication**: - - **username**: replication username; user will be created during initialization. - - **password**: replication password; user will be created during initialization. + - **username**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication + - **password**: replication password; the user will be created during initialization. - **callbacks**: callback scripts to run on certain actions. Patroni will pass the action, role and cluster name. (See scripts/aws.py as an example of how to write them.) - **on\_reload**: run this script when configuration reload is triggered. - **on\_restart**: run this script when the cluster restarts. @@ -58,20 +58,24 @@ PostgreSQL - **on\_stop**: run this script when the cluster stops. - **connect\_address**: IP address + port through which Postgres is accessible from other nodes and applications. - **create\_replica\_methods**: an ordered list of the create methods for turning a Patroni node into a new replica. "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its own config item. -- **data\_dir**: file path to initialize and store Postgres data files. +- **data\_dir**: The location of the Postgres data directory, either existing or to be initialized by Patroni. - **listen**: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node. +- **pgpass**: path to the [.pgpass password file](https://www.postgresql.org/docs/current/static/libpq-pgpass.html). Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni. - **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower. - **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work. - **replica\_method** for each create_replica_method other than basebackup, you would add a configuration section of the same name. At a minimum, this should include "command" with a full path to the actual script to be executed. Other configuration parameters will be passed along to the script in the form "parameter=value". REST API -------- -- **connect\_address**: IP address and port through which restapi is accessible. +- **connect\_address**: IP address and port to access the REST API. - **listen**: IP address and port that Patroni will listen to, to provide health-check information for HAProxy. -- **Optional**: - - **auth**: 'username:password' to protect dangerous REST API endpoints. - - **certfile**: Specifies a file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL. - - **keyfile**: Specifies a file with the secret key in the PEM format. +- **Optional**: + - **authentication**: + - **username**: Basic-auth username to protect unsafe REST API endpoints. + - **password**: Basic-auth password to protect unsafe REST API endpoints. + + - **certfile**: Specifies the file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL. + - **keyfile**: Specifies the file with the secret key in the PEM format. ZooKeeper ---------- From e9be5e846290a7bdaaebdda4e5b3f97c9f15252f Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 9 Jun 2016 11:40:10 +0200 Subject: [PATCH 5/8] Configure exhibitor port via ENV --- docs/ENVIRONMENT.rst | 1 + patroni/config.py | 12 +++++++----- tests/test_config.py | 2 ++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst index 70aae806..d5ea6c1e 100644 --- a/docs/ENVIRONMENT.rst +++ b/docs/ENVIRONMENT.rst @@ -31,6 +31,7 @@ Etcd Exhibitor --------- - **PATRONI\_EXHIBITOR\_HOSTS**: initial list of Exhibitor (ZooKeeper) nodes in format: ['host1', 'host2', 'etc...' ]. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes. +- **PATRONI\_EXHIBITOR\_PORT**: Exhibitor port. PostgreSQL ---------- diff --git a/patroni/config.py b/patroni/config.py index e4a4b390..07fd508f 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -8,7 +8,7 @@ from collections import defaultdict from copy import deepcopy from patroni.dcs import ClusterConfig from patroni.postgresql import Postgresql -from patroni.utils import deep_compare, patch_config +from patroni.utils import deep_compare, parse_int, patch_config logger = logging.getLogger(__name__) @@ -219,10 +219,13 @@ class Config(object): if param.startswith('PATRONI_'): name, suffix = (param[8:].rsplit('_', 1) + [''])[:2] if name and suffix: - # PATRONI_(ETCD|CONSUL|ZOOKEEPER|...)_HOSTS? - if suffix in ('HOST', 'HOSTS') and '_' not in name: + # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT) + if suffix in ('HOST', 'HOSTS', 'PORT') and '_' not in name: value = os.environ.pop(param) - value = value if suffix == 'HOST' else value and _parse_list(value) + if suffix == 'PORT': + value = value and parse_int(value) + elif suffix == 'HOSTS': + value = value and _parse_list(value) if value: ret[name.lower()][suffix.lower()] = value # PATRONI__PASSWORD=, PATRONI__OPTIONS= @@ -252,7 +255,6 @@ class Config(object): elif name not in config: config[name] = deepcopy(value) if value else {} - # restapi server expects to get restapi.auth = 'username:password' if 'authentication' in config['restapi']: restapi = config['restapi'] diff --git a/tests/test_config.py b/tests/test_config.py index d85d4f3a..e6d212d6 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -36,6 +36,8 @@ class TestConfig(unittest.TestCase): 'PATRONI_ETCD_HOST': '127.0.0.1:2379', 'PATRONI_CONSUL_HOST': '127.0.0.1:8500', 'PATRONI_ZOOKEEPER_HOSTS': 'host1,host2', + 'PATRONI_EXHIBITOR_HOSTS': 'host1,host2', + 'PATRONI_EXHIBITOR_PORT': '8181', 'PATRONI_foo_HOSTS': '[host1,host2', # Exception in parse_list 'PATRONI_SUPERUSER_USERNAME': 'postgres', 'PATRONI_SUPERUSER_PASSWORD': 'zalando', From 7244739e2637f5d889dc3ca4bbdeb356f41fb1d2 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 9 Jun 2016 12:10:37 +0200 Subject: [PATCH 6/8] Fix link to the libpq-pgpass.html --- docs/ENVIRONMENT.rst | 2 +- docs/SETTINGS.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst index d5ea6c1e..1db8dadf 100644 --- a/docs/ENVIRONMENT.rst +++ b/docs/ENVIRONMENT.rst @@ -38,7 +38,7 @@ PostgreSQL - **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node. - **PATRONI\_POSTGRESQL\_CONNECT\_ADDRESS**: IP address + port through which Postgres is accessible from other nodes and applications. - **PATRONI\_POSTGRESQL\_DATA\_DIR**: The location of the Postgres data directory, either existing or to be initialized by Patroni. -- **PATRONI\_POSTGRESQL\_PGPASS**: path to the [.pgpass password file](https://www.postgresql.org/docs/current/static/libpq-pgpass.html). Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni. +- **PATRONI\_POSTGRESQL\_PGPASS**: path to the `.pgpass `__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni. - **PATRONI\_REPLICATION\_USERNAME**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication - **PATRONI\_REPLICATION\_PASSWORD**: replication password; the user will be created during initialization. - **PATRONI\_SUPERUSER\_USERNAME**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres. Also this user is used by pg_rewind. diff --git a/docs/SETTINGS.rst b/docs/SETTINGS.rst index 8e46e952..86f21d5c 100644 --- a/docs/SETTINGS.rst +++ b/docs/SETTINGS.rst @@ -66,7 +66,7 @@ PostgreSQL - **create\_replica\_methods**: an ordered list of the create methods for turning a Patroni node into a new replica. "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its own config item. - **data\_dir**: The location of the Postgres data directory, either existing or to be initialized by Patroni. - **listen**: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node. -- **pgpass**: path to the [.pgpass password file](https://www.postgresql.org/docs/current/static/libpq-pgpass.html). Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni. +- **pgpass**: path to the `.pgpass `__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni. - **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower. - **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work. - **replica\_method** for each create_replica_method other than basebackup, you would add a configuration section of the same name. At a minimum, this should include "command" with a full path to the actual script to be executed. Other configuration parameters will be passed along to the script in the form "parameter=value". From 49efb371f9b46e0552d32c999dd03b80de27f678 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 9 Jun 2016 14:44:29 +0200 Subject: [PATCH 7/8] Make it possible to work without config.yml Most of the basic configuration could be done via ENV --- patroni/__init__.py | 20 +++----------------- patroni/config.py | 30 ++++++++++++++++++++++-------- tests/test_config.py | 11 +++++++++-- tests/test_ha.py | 10 ++++++---- tests/test_patroni.py | 12 ++---------- 5 files changed, 42 insertions(+), 41 deletions(-) diff --git a/patroni/__init__.py b/patroni/__init__.py index 84a1018d..c48eab40 100644 --- a/patroni/__init__.py +++ b/patroni/__init__.py @@ -1,7 +1,5 @@ import logging -import os import signal -import sys import time from patroni.api import RestApiServer @@ -17,11 +15,10 @@ logger = logging.getLogger(__name__) class Patroni(object): - PATRONI_CONFIG_VARIABLE = 'PATRONI_CONFIGURATION' - def __init__(self, config_file=None, config_env=None): + def __init__(self): self.version = __version__ - self.config = Config(config_file=config_file, config_env=config_env) + self.config = Config() self.dcs = get_dcs(self.config) self.load_dynamic_configuration() @@ -116,18 +113,7 @@ def main(): logging.getLogger('requests').setLevel(logging.WARNING) setup_signal_handlers() - # Patroni reads the configuration from the command-line argument if it exists, and from the environment otherwise. - config_env = False - config_file = len(sys.argv) >= 2 and os.path.isfile(sys.argv[1]) and sys.argv[1] - if not config_file: - config_env = os.environ.pop(Patroni.PATRONI_CONFIG_VARIABLE, None) - if config_env is None: - print('Usage: {0} config.yml'.format(sys.argv[0])) - print('\tPatroni may also read the configuration from the {} environment variable'. - format(Patroni.PATRONI_CONFIG_VARIABLE)) - return - - patroni = Patroni(config_file, config_env) + patroni = Patroni() try: patroni.run() except KeyboardInterrupt: diff --git a/patroni/config.py b/patroni/config.py index 07fd508f..d982842a 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -1,6 +1,7 @@ import json import logging import os +import sys import tempfile import yaml @@ -33,6 +34,9 @@ class Config(object): to work with it as with the old `config` object. """ + PATRONI_ENV_PREFIX = 'PATRONI_' + PATRONI_CONFIG_VARIABLE = PATRONI_ENV_PREFIX + 'CONFIGURATION' + __CACHE_FILENAME = 'patroni.dynamic.json' __DEFAULT_CONFIG = { 'ttl': 30, 'loop_wait': 10, 'retry_timeout': 10, @@ -42,15 +46,25 @@ class Config(object): } } - def __init__(self, config_file=None, config_env=None): - self._config_file = None if config_env else config_file + def __init__(self): self._modify_index = -1 self._dynamic_configuration = {} - if config_env: - self._local_configuration = yaml.safe_load(config_env) - else: - self.__environment_configuration = self._build_environment_configuration() + + self.__environment_configuration = self._build_environment_configuration() + + # Patroni reads the configuration from the command-line argument if it exists, otherwise from the environment + self._config_file = len(sys.argv) >= 2 and os.path.isfile(sys.argv[1]) and sys.argv[1] + if self._config_file: self._local_configuration = self._load_config_file() + else: + config_env = os.environ.pop(self.PATRONI_CONFIG_VARIABLE, None) + self._local_configuration = config_env and yaml.safe_load(config_env) or self.__environment_configuration + if not self._local_configuration: + print('Usage: {0} config.yml'.format(sys.argv[0])) + print('\tPatroni may also read the configuration from the {0} environment variable'. + format(self.PATRONI_CONFIG_VARIABLE)) + exit(1) + self.__effective_configuration = self._build_effective_configuration(self._dynamic_configuration, self._local_configuration) self._data_dir = self.__effective_configuration['postgresql']['data_dir'] @@ -168,7 +182,7 @@ class Config(object): ret = defaultdict(dict) def _popenv(name): - return os.environ.pop('PATRONI_' + name.upper(), None) + return os.environ.pop(Config.PATRONI_ENV_PREFIX + name.upper(), None) for param in ('name', 'namespace', 'scope'): value = _popenv(param) @@ -216,7 +230,7 @@ class Config(object): return None for param in list(os.environ.keys()): - if param.startswith('PATRONI_'): + if param.startswith(Config.PATRONI_ENV_PREFIX): name, suffix = (param[8:].rsplit('_', 1) + [''])[:2] if name and suffix: # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT) diff --git a/tests/test_config.py b/tests/test_config.py index e6d212d6..a8a5a6c4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,6 @@ import os import unittest +import sys from mock import MagicMock, Mock, patch from patroni.config import Config @@ -12,7 +13,12 @@ class TestConfig(unittest.TestCase): @patch('json.load', Mock(side_effect=Exception)) @patch.object(builtins, 'open', MagicMock()) def setUp(self): - self.config = Config(config_env='restapi: {}\npostgresql: {data_dir: foo}') + sys.argv = ['patroni.py'] + os.environ[Config.PATRONI_CONFIG_VARIABLE] = 'restapi: {}\npostgresql: {data_dir: foo}' + self.config = Config() + + def test_no_config(self): + self.assertRaises(SystemExit, Config) @patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)) def test_set_dynamic_configuration(self): @@ -46,7 +52,8 @@ class TestConfig(unittest.TestCase): 'PATRONI_admin_PASSWORD': 'admin', 'PATRONI_admin_OPTIONS': 'createrole,createdb' }) - config = Config(config_file='postgres0.yml') + sys.argv = ['patroni.py', 'postgres0.yml'] + config = Config() with patch.object(Config, '_load_config_file', Mock(return_value={'restapi': {}})): with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)): self.assertRaises(Exception, config.reload_local_configuration, True) diff --git a/tests/test_ha.py b/tests/test_ha.py index 59be45cd..4f691212 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -1,7 +1,8 @@ -import etcd -import unittest import datetime +import etcd +import os import pytz +import unittest from mock import Mock, MagicMock, patch from patroni.config import Config @@ -49,7 +50,7 @@ def get_cluster_initialized_with_only_leader(failover=None): class MockPatroni(object): def __init__(self, p, d): - self.config = Config(config_env=""" + os.environ[Config.PATRONI_CONFIG_VARIABLE] = """ restapi: listen: 0.0.0.0:8008 bootstrap: @@ -68,7 +69,8 @@ zookeeper: exhibitor: hosts: [localhost] port: 8181 -""") +""" + self.config = Config() self.postgresql = p self.dcs = d self.api = Mock() diff --git a/tests/test_patroni.py b/tests/test_patroni.py index b685f2ae..f9246cac 100644 --- a/tests/test_patroni.py +++ b/tests/test_patroni.py @@ -1,5 +1,4 @@ import etcd -import os import sys import time import unittest @@ -34,7 +33,8 @@ class TestPatroni(unittest.TestCase): RestApiServer.socket = 0 with patch.object(etcd.Client, 'machines') as mock_machines: mock_machines.__get__ = Mock(return_value=['http://remotehost:2379']) - self.p = Patroni('postgres0.yml') + sys.argv = ['patroni.py', 'postgres0.yml'] + self.p = Patroni() @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(side_effect=[None, DCSError('foo'), None])) def test_load_dynamic_configuration(self): @@ -47,7 +47,6 @@ class TestPatroni(unittest.TestCase): @patch.object(etcd.Client, 'machines') def test_patroni_main(self, mock_machines): with patch('subprocess.call', Mock(return_value=1)): - _main() sys.argv = ['patroni.py', 'postgres0.yml'] mock_machines.__get__ = Mock(return_value=['http://remotehost:2379']) @@ -55,13 +54,6 @@ class TestPatroni(unittest.TestCase): self.assertRaises(SleepException, _main) with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())): _main() - sys.argv = ['patroni.py'] - # read the content of the yaml configuration file into the environment variable - # in order to test how does patroni handle the configuration passed from the environment. - with open('postgres0.yml', 'r') as f: - os.environ[Patroni.PATRONI_CONFIG_VARIABLE] = f.read() - with patch.object(Patroni, 'run', Mock(side_effect=SleepException())): - self.assertRaises(SleepException, _main) @patch('patroni.config.Config.save_cache', Mock()) @patch('patroni.config.Config.reload_local_configuration', Mock(return_value=True)) From 9ecff0f64d25ddbb9426423b692f6382c49867a5 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 10 Jun 2016 12:35:04 +0200 Subject: [PATCH 8/8] Bugfixes * GET /config was returning latesy "correct" version of dynamic configuration. * PATCH /config was breaking when trying to patch not dict with dict --- patroni/api.py | 6 +++++- patroni/utils.py | 6 +++++- tests/test_api.py | 10 ++++++---- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/patroni/api.py b/patroni/api.py index 420067ca..7000fc2f 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -106,7 +106,11 @@ class RestApiHandler(BaseHTTPRequestHandler): self._write_status_response(200, response) def do_GET_config(self): - self._write_json_response(200, self.server.patroni.config.dynamic_configuration) + cluster = self.server.patroni.ha.dcs.cluster or self.server.patroni.ha.dcs.get_cluster() + if cluster.config: + self._write_json_response(200, cluster.config.data) + else: + self.send_error(502) def _read_json_content(self): if 'content-length' not in self.headers: diff --git a/patroni/utils.py b/patroni/utils.py index 9c63c768..13869098 100644 --- a/patroni/utils.py +++ b/patroni/utils.py @@ -70,7 +70,11 @@ def patch_config(config, data): is_changed = True elif name in config: if isinstance(value, dict): - if patch_config(config[name], value): + if isinstance(config[name], dict): + if patch_config(config[name], value): + is_changed = True + else: + config[name] = value is_changed = True elif str(config[name]) != str(value): config[name] = value diff --git a/tests/test_api.py b/tests/test_api.py index 079fa7f7..8483299b 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -118,9 +118,11 @@ class TestRestApiHandler(unittest.TestCase): self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0')) MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0\nAuthorization:') - @patch.object(MockPatroni, 'config') - def test_do_GET_config(self, mock_config): - mock_config.dynamic_configuration = {} + @patch.object(MockHa, 'dcs') + def test_do_GET_config(self, mock_dcs): + mock_dcs.cluster.config.data = {} + self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /config')) + mock_dcs.cluster.config = None self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /config')) @patch.object(MockHa, 'dcs') @@ -132,7 +134,7 @@ class TestRestApiHandler(unittest.TestCase): request += '\nContent-Length: ' self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '34\n\n{"postgresql":{"use_slots":false}}')) config['ttl'] = 5 - config['postgresql'].update({'use_slots': True, "parameters": None}) + config['postgresql'].update({'use_slots': {'foo': True}, "parameters": None}) config = json.dumps(config) request += str(len(config)) + '\n\n' + config MockRestApiServer(RestApiHandler, request)