Implement possibility to configure Patroni via environment

README.rst

@@ -75,7 +75,13 @@ run:
 YAML Configuration
 ===============
 
-Go `here <https://github.com/zalando/patroni/blob/master/SETTINGS.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__. 
+Go `here <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__. 
+
+=========================
+Environment Configuration
+=========================
+
+Go `here <https://github.com/zalando/patroni/blob/master/docs/ENVIRONMENT.rst>`__ for comprehensive information about configuring(overriding) settings via environment variables.
 
 ===============
 Replication Choices

docs/ENVIRONMENT.rst

@@ -0,0 +1,52 @@
+==================================
+Environment Configuration Settings
+==================================
+
+Some of configuration parameters defined in the configuration file is possible to override via Environment variables
+This document list all possible environment variables handled by Patroni.
+Environment variable always takes precedence on configuration file.
+
+Global/Universal
+----------------
+-  **PATRONI\_NAME**: name of the node where the current instance of Patroni is running. Must be unique for the cluster.
+-  **PATRONI\_NAMESPACE**: path within configuration store where Patroni will keep information about cluster. Default value: "/service"
+-  **PATRONI\_SCOPE**: cluster name
+
+Bootstrap configuration
+-----------------------
+It is possible to define users which will be created right after initializing of a new cluster by defining following environment variables:
+-  **PATRONI\_<username>\_PASSWORD='<password>'**
+-  **PATRONI\_<username>\_OPTIONS='list,of,options'**
+Example: defining of `PATRONI\_admin\_PASSWORD=admin` `PATRONI\_admin\_OPTIONS='createrole,createdb'` will cause creation of `admin` user which is allowed to create other users and databases
+
+Consul
+------
+-  **PATRONI\_CONSUL\_HOST**: the host:port for the Consul endpoint.
+
+Etcd
+----
+-  **PATRONI\_ETCD\_HOST**: the host:port for the etcd endpoint.
+
+PostgreSQL
+----------
+-  **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
+-  **PATRONI\_POSTGRESQL\_CONNECT\_ADDRESS**: IP address + port through which Postgres is accessible from other nodes and applications.
+-  **PATRONI\_POSTGRESQL\_DATA\_DIR**: file path to initialize and store Postgres data files.
+-  **PATRONI\_POSTGRESQL\_PGPASS**: path to `pgpass` file which would be created by Patroni when it is necessary (for example, before executing pg\_basebackup). This locations must be accessible for writing by Patroni.
+-  **PATRONI\_REPLICATION\_USERNAME**: replication username; user will be created during initialization. Replicas will use this user to access master via streaming replication
+-  **PATRONI\_REPLICATION\_PASSWORD**: replication password; user will be created during initialization.
+-  **PATRONI\_SUPERUSER\_USERNAME**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres. Also this user is used by pg_rewind.
+-  **PATRONI\_SUPERUSER\_PASSWORD**: password for the superuser, set during initialization (initdb).
+
+REST API
+-------- 
+-  **PATRONI\_RESTAPI\_CONNECT\_ADDRESS**: IP address and port through which restapi is accessible.
+-  **PATRONI\_RESTAPI\_LISTEN**: IP address and port that Patroni will listen to, to provide health-check information for HAProxy.
+-  **PATRONI\_RESTAPI\_USERNAME**: Basic-auth username to protect dangerous REST API endpoints.
+-  **PATRONI\_RESTAPI\_PASSWORD**: Basic-auth password to protect dangerous REST API endpoints.
+-  **PATRONI\_RESTAPI\_CERTFILE**: Specifies a file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL.
+-  **PATRONI\_RESTAPI\_KEYFILE**: Specifies a file with the secret key in the PEM format.
+
+ZooKeeper
+---------
+-  **PATRONI\_ZOOKEEPER\_HOSTS**: comma separated list of ZooKeeper cluster members: 'host1:port1,host2:port2,etc...'

patroni/__init__.py

@@ -120,7 +120,7 @@ def main():
     config_env = False
     config_file = len(sys.argv) >= 2 and os.path.isfile(sys.argv[1]) and sys.argv[1]
     if not config_file:
-        config_env = os.environ.get(Patroni.PATRONI_CONFIG_VARIABLE)
+        config_env = os.environ.pop(Patroni.PATRONI_CONFIG_VARIABLE, None)
         if config_env is None:
             print('Usage: {0} config.yml'.format(sys.argv[0]))
             print('\tPatroni may also read the configuration from the {} environment variable'.

patroni/api.py

@@ -9,7 +9,7 @@ import datetime
 import pytz
 
 from patroni.exceptions import PostgresConnectionException
-from patroni.utils import deep_compare, Retry, RetryFailedError
+from patroni.utils import deep_compare, patch_config, Retry, RetryFailedError
 from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 from six.moves.socketserver import ThreadingMixIn
 from threading import Thread
@@ -108,25 +108,6 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_GET_config(self):
         self._write_json_response(200, self.server.patroni.config.dynamic_configuration)
 
-    @staticmethod
-    def _patch_config(config, data):
-        is_changed = False
-        for name, value in data.items():
-            if value is None:
-                if config.pop(name, None) is not None:
-                    is_changed = True
-            elif name in config:
-                if isinstance(value, dict):
-                    if RestApiHandler._patch_config(config[name], value):
-                        is_changed = True
-                elif str(config[name]) != str(value):
-                    config[name] = value
-                    is_changed = True
-            else:
-                config[name] = value
-                is_changed = True
-        return is_changed
-
     def _read_json_content(self):
         if 'content-length' not in self.headers:
             return self.send_error(411)
@@ -145,7 +126,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         if request:
             cluster = self.server.patroni.ha.dcs.get_cluster()
             data = cluster.config.data.copy()
-            if RestApiHandler._patch_config(data, request):
+            if patch_config(data, request):
                 value = json.dumps(data, separators=(',', ':'))
                 if not self.server.patroni.ha.dcs.set_config_value(value, cluster.config.index):
                     return self.send_error(409)

patroni/config.py

@@ -4,10 +4,11 @@ import os
 import tempfile
 import yaml
 
+from collections import defaultdict
 from copy import deepcopy
 from patroni.dcs import ClusterConfig
 from patroni.postgresql import Postgresql
-from patroni.utils import deep_compare
+from patroni.utils import deep_compare, patch_config
 
 logger = logging.getLogger(__name__)
 
@@ -45,7 +46,11 @@ class Config(object):
         self._config_file = None if config_env else config_file
         self._modify_index = -1
         self._dynamic_configuration = {}
-        self._local_configuration = yaml.safe_load(config_env) if config_env else self._load_config_file()
+        if config_env:
+            self._local_configuration = yaml.safe_load(config_env)
+        else:
+            self.__environment_configuration = self._build_environment_configuration()
+            self._local_configuration = self._load_config_file()
         self.__effective_configuration = self._build_effective_configuration(self._dynamic_configuration,
                                                                              self._local_configuration)
         self._data_dir = self.__effective_configuration['postgresql']['data_dir']
@@ -62,8 +67,11 @@ class Config(object):
         return deepcopy(self._dynamic_configuration)
 
     def _load_config_file(self):
+        """Loads config.yaml from filesystem and applies some values which were set via ENV"""
         with open(self._config_file) as f:
-            return yaml.safe_load(f)
+            config = yaml.safe_load(f)
+            patch_config(config, self.__environment_configuration)
+            return config
 
     def _load_cache(self):
         if os.path.isfile(self._cache_file):
@@ -151,10 +159,86 @@ class Config(object):
                         config['postgresql'][name].update(self._process_postgresql_parameters(value))
                     elif name not in ('connect_address', 'listen', 'data_dir', 'pgpass', 'authentication'):
                         config['postgresql'][name] = deepcopy(value)
-            elif name in config:
+            elif name in config:  # only variables present in __DEFAULT_CONFIG allowed to be overriden from DCS
                 config[name] = int(value)
         return config
 
+    @staticmethod
+    def _build_environment_configuration():
+        ret = defaultdict(dict)
+        def _popenv(name):
+            return os.environ.pop('PATRONI_' + name.upper(), None)
+
+        for param in ('name', 'namespace', 'scope'):
+            value = _popenv(param)
+            if value:
+                ret[param] = value
+
+        def _set_section_values(section, params):
+            for param in params:
+                value = _popenv(section + '_' + param)
+                if value:
+                    ret[section][param] = value
+
+        _set_section_values('restapi', ['listen', 'connect_address', 'certfile', 'keyfile'])
+        _set_section_values('postgresql', ['listen', 'connect_address', 'data_dir', 'pgpass'])
+
+        def _get_auth(name):
+            ret = {}
+            for param in ('username', 'password'):
+                value = _popenv(name + '_' + param)
+                if value:
+                    ret[param] = value
+            return len(ret) == 2 and ret or None
+
+        restapi_auth = _get_auth('restapi')
+        if restapi_auth:
+            ret['restapi']['authentication'] = restapi_auth
+
+        authentication = {}
+        for user_type in ('replication', 'superuser'):
+            entry = _get_auth(user_type)
+            if entry:
+                authentication[user_type] = entry
+
+        if authentication:
+            ret['postgresql']['authentication'] = authentication
+
+        users = {}
+
+        def _parse_list(value):
+            if not (value.strip().startswith('-') or '[' in value):
+                value = '[{0}]'.format(value)
+            try:
+                return yaml.safe_load(value)
+            except Exception:
+                return None
+
+        for param in list(os.environ.keys()):
+            if param.startswith('PATRONI_'):
+                name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
+                if name and suffix:
+                    # PATRONI_(ETCD|CONSUL|ZOOKEEPER|...)_HOSTS?
+                    if suffix in ('HOST', 'HOSTS') and '_' not in name:
+                        value = os.environ.pop(param)
+                        value = value if suffix == 'HOST' else value and _parse_list(value)
+                        if value:
+                            ret[name.lower()][suffix.lower()] = value
+                    # PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>
+                    # CREATE USER "<username>" WITH <OPTIONS> PASSWORD '<password>'
+                    elif suffix == 'PASSWORD':
+                        password = os.environ.pop(param)
+                        if password:
+                            users[name] = {'password': password}
+                            options = os.environ.pop(param[:-9] + '_OPTIONS', None)
+                            options = options and _parse_list(options)
+                            if options:
+                                users[name]['options'] = options
+        if users:
+            ret['bootstrap']['users'] = users
+
+        return ret
+
     def _build_effective_configuration(self, dynamic_configuration, local_configuration):
         config = self._safe_copy_dynamic_configuration(dynamic_configuration)
         for name, value in local_configuration.items():
@@ -167,6 +251,11 @@ class Config(object):
             elif name not in config:
                 config[name] = deepcopy(value) if value else {}
 
+        if 'authentication' in config['restapi']:
+            restapi = config['restapi']
+            auth = restapi['authentication']
+            restapi['auth'] = '{0}:{1}'.format(auth['username'], auth['password'])
+
         pg_config = config['postgresql']
 
         # special treatment for old config

patroni/utils.py

@@ -60,6 +60,27 @@ def deep_compare(obj1, obj2):
     return True
 
 
+def patch_config(config, data):
+    """recursively 'patch' `config` with `data`
+    :returns: `!True` if the `config` was changed"""
+    is_changed = False
+    for name, value in data.items():
+        if value is None:
+            if config.pop(name, None) is not None:
+                is_changed = True
+        elif name in config:
+            if isinstance(value, dict):
+                if patch_config(config[name], value):
+                    is_changed = True
+            elif str(config[name]) != str(value):
+                config[name] = value
+                is_changed = True
+        else:
+            config[name] = value
+            is_changed = True
+    return is_changed
+
+
 def parse_bool(value):
     """
     >>> parse_bool(1)

postgres0.yml

@@ -4,6 +4,9 @@ name: postgresql0
 
 restapi:
   listen: 127.0.0.1:8008
+#  authentication:
+#    username: username
+#    password: password
   connect_address: 127.0.0.1:8008
 
 etcd:

tests/test_config.py

@@ -1,3 +1,4 @@
+import os
 import unittest
 
 from mock import MagicMock, Mock, patch
@@ -11,15 +12,40 @@ class TestConfig(unittest.TestCase):
     @patch('json.load', Mock(side_effect=Exception))
     @patch.object(builtins, 'open', MagicMock())
     def setUp(self):
-        self.config = Config(config_env='postgresql: {data_dir: foo}')
+        self.config = Config(config_env='restapi: {}\npostgresql: {data_dir: foo}')
 
     @patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception))
     def test_set_dynamic_configuration(self):
         self.assertIsNone(self.config.set_dynamic_configuration({'foo': 'bar'}))
 
     def test_reload_local_configuration(self):
+        os.environ.update({
+            'PATRONI_NAME': 'postgres0',
+            'PATRONI_NAMESPACE': '/patroni/',
+            'PATRONI_SCOPE': 'batman2',
+            'PATRONI_RESTAPI_USERNAME': 'username',
+            'PATRONI_RESTAPI_PASSWORD': 'password',
+            'PATRONI_RESTAPI_LISTEN': '0.0.0.0:8008',
+            'PATRONI_RESTAPI_CONNECT_ADDRESS': '127.0.0.1:8008',
+            'PATRONI_RESTAPI_CERTFILE': '/certfile',
+            'PATRONI_RESTAPI_KEYFILE': '/keyfile',
+            'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432',
+            'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432',
+            'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
+            'PATRONI_POSTGRESQL_PGPASS': '/tmp/pgpass0',
+            'PATRONI_ETCD_HOST': '127.0.0.1:2379',
+            'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
+            'PATRONI_ZOOKEEPER_HOSTS': 'host1,host2',
+            'PATRONI_foo_HOSTS': '[host1,host2',  # Exception in parse_list
+            'PATRONI_SUPERUSER_USERNAME': 'postgres',
+            'PATRONI_SUPERUSER_PASSWORD': 'zalando',
+            'PATRONI_REPLICATION_USERNAME': 'replicator',
+            'PATRONI_REPLICATION_PASSWORD': 'rep-pass',
+            'PATRONI_admin_PASSWORD': 'admin',
+            'PATRONI_admin_OPTIONS': 'createrole,createdb'
+        })
         config = Config(config_file='postgres0.yml')
-        with patch.object(Config, '_load_config_file', Mock(return_value={})):
+        with patch.object(Config, '_load_config_file', Mock(return_value={'restapi': {}})):
             with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
                 self.assertRaises(Exception, config.reload_local_configuration, True)
             self.assertTrue(config.reload_local_configuration(True))

tests/test_ha.py

@@ -50,6 +50,8 @@ class MockPatroni(object):
 
     def __init__(self, p, d):
         self.config = Config(config_env="""
+restapi:
+  listen: 0.0.0.0:8008
 bootstrap:
   users:
     replicator:

tests/test_patroni.py

@@ -61,7 +61,6 @@ class TestPatroni(unittest.TestCase):
                 os.environ[Patroni.PATRONI_CONFIG_VARIABLE] = f.read()
             with patch.object(Patroni, 'run', Mock(side_effect=SleepException())):
                 self.assertRaises(SleepException, _main)
-            del os.environ[Patroni.PATRONI_CONFIG_VARIABLE]
 
     @patch('patroni.config.Config.save_cache', Mock())
     @patch('patroni.config.Config.reload_local_configuration', Mock(return_value=True))