From 5ef01cfdfae2f50239d82ab096809663c7d29c67 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 24 Aug 2017 07:56:12 +0200 Subject: [PATCH] Advanced configuration for Consul (#506) * possibility to specify client certs and cacert * possibility to specify token * compatibility with python-consul-0.7.1 --- docs/ENVIRONMENT.rst | 8 +++++ docs/SETTINGS.rst | 11 ++++++- patroni/config.py | 4 +-- patroni/dcs/consul.py | 72 ++++++++++++++++++++++++++++++++++--------- requirements.txt | 2 +- tests/test_consul.py | 4 +++ 6 files changed, 82 insertions(+), 19 deletions(-) diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst index 3a8494f6..275e9c66 100644 --- a/docs/ENVIRONMENT.rst +++ b/docs/ENVIRONMENT.rst @@ -25,6 +25,14 @@ Example: defining ``PATRONI_admin_PASSWORD=strongpasswd`` and ``PATRONI_admin_OP Consul ------ - **PATRONI\_CONSUL\_HOST**: the host:port for the Consul endpoint. +- **PATRONI\_CONSUL\_URL**: url for the Consul, in format: http(s)://host:port +- **PATRONI\_CONSUL\_PORT**: (optional) Consul port +- **PATRONI\_CONSUL\_SCHEME**: (optional) **http** or **https**, defaults to **http** +- **PATRONI\_CONSUL\_TOKEN**: (optional) ACL token +- **PATRONI\_CONSUL\_VERIFY**: (optional) whether to verify the SSL certificate for HTTPS requests +- **PATRONI\_CONSUL\_CACERT**: (optional) The ca certificate. If pressent it will enable validation. +- **PATRONI\_CONSUL\_CERT**: (optional) File with the client certificate +- **PATRONI\_CONSUL\_KEY**: (optional) File with the client key. Can be empty if the key is part of certificate. Etcd ---- diff --git a/docs/SETTINGS.rst b/docs/SETTINGS.rst index 5e6ef774..7946c07c 100644 --- a/docs/SETTINGS.rst +++ b/docs/SETTINGS.rst @@ -45,7 +45,16 @@ Bootstrap configuration Consul ------ -- **host**: the host:port for the Consul endpoint. +Most of the parameters are optional, but you have to specify one of the **host** or **url** +- **host**: the host:port for the Consul endpoint, in format: http(s)://host:port +- **url**: url for the Consul endpoint +- **port**: (optional) Consul port +- **scheme**: (optional) **http** or **https**, defaults to **http** +- **token**: (optional) ACL token +- **verify** (optional) whether to verify the SSL certificate for HTTPS requests +- **cacert**: (optional) The ca certificate. If pressent it will enable validation. +- **cert**: (optional) file with the client certificate +- **key**: (optional) file with the client key. Can be empty if the key is part of **cert**. Etcd ---- diff --git a/patroni/config.py b/patroni/config.py index 7e740810..42f47ddb 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -242,8 +242,8 @@ class Config(object): name, suffix = (param[8:].rsplit('_', 1) + [''])[:2] if name and suffix: # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..) - if suffix in ('HOST', 'HOSTS', 'PORT', 'SRV', 'URL', 'PROXY', 'CACERT', 'CERT', 'KEY') \ - and '_' not in name: + if suffix in ('HOST', 'HOSTS', 'PORT', 'SRV', 'URL', 'PROXY', 'CACERT', 'CERT', 'KEY', + 'VERIFY', 'TOKEN') and '_' not in name: value = os.environ.pop(param) if suffix == 'PORT': value = value and parse_int(value) diff --git a/patroni/dcs/consul.py b/patroni/dcs/consul.py index df907803..840d5992 100644 --- a/patroni/dcs/consul.py +++ b/patroni/dcs/consul.py @@ -2,15 +2,16 @@ from __future__ import absolute_import import logging import os import socket +import ssl import time import urllib3 from consul import ConsulException, NotFound, base from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState from patroni.exceptions import DCSError -from patroni.utils import Retry, RetryFailedError +from patroni.utils import parse_bool, Retry, RetryFailedError from urllib3.exceptions import HTTPError -from six.moves.urllib.parse import urlencode +from six.moves.urllib.parse import urlencode, urlparse from six.moves.http_client import HTTPException logger = logging.getLogger(__name__) @@ -26,14 +27,23 @@ class ConsulInternalError(ConsulException): class HTTPClient(object): - def __init__(self, host='127.0.0.1', port=8500, scheme='http', verify=True, timeout=10): - self.host = host - self.port = port - self.scheme = scheme - self.verify = verify - self.set_read_timeout(timeout) - self.base_uri = '{0}://{1}:{2}'.format(self.scheme, self.host, self.port) - self.http = urllib3.PoolManager(num_pools=10) + def __init__(self, host='127.0.0.1', port=8500, scheme='http', verify=True, cert=None, ca_cert=None): + self._read_timeout = 10 + self.base_uri = '{0}://{1}:{2}'.format(scheme, host, port) + kwargs = {} + if cert: + if isinstance(cert, tuple): + # Key and cert are separate + kwargs['cert_file'] = cert[0] + kwargs['key_file'] = cert[1] + else: + # combined certificate + kwargs['cert_file'] = cert + if ca_cert: + kwargs['ca_certs'] = ca_cert + if verify or ca_cert: + kwargs['cert_reqs'] = ssl.CERT_REQUIRED + self.http = urllib3.PoolManager(num_pools=10, **kwargs) self._ttl = None def set_read_timeout(self, timeout): @@ -78,9 +88,18 @@ class HTTPClient(object): class ConsulClient(base.Consul): - @staticmethod - def connect(host, port, scheme, verify=True): - return HTTPClient(host, port, scheme, verify) + def __init__(self, *args, **kwargs): + self._cert = kwargs.pop('cert', None) + self._ca_cert = kwargs.pop('ca_cert', None) + super(ConsulClient, self).__init__(*args, **kwargs) + + def connect(self, *args, **kwargs): + kwargs.update(dict(zip(['host', 'port', 'scheme', 'verify'], args))) + if self._cert: + kwargs['cert'] = self._cert + if self._ca_cert: + kwargs['ca_cert'] = self._ca_cert + return HTTPClient(**kwargs) def catch_consul_errors(func): @@ -104,8 +123,31 @@ class Consul(AbstractDCS): HTTPError, socket.error, socket.timeout)) self._my_member_data = None - host, port = config.get('host', '127.0.0.1:8500').split(':') - self._client = ConsulClient(host=host, port=port) + kwargs = {} + if 'url' in config: + r = urlparse(config['url']) + config.update({'scheme': r.scheme, 'host': r.hostname, 'port': r.port or 8500}) + elif 'host' in config: + host, port = (config.get('host', '127.0.0.1:8500') + ':8500').split(':')[:2] + config['host'] = host + if 'port' not in config: + config['port'] = int(port) + + if config.get('cacert'): + config['ca_cert'] = config.pop('cacert') + + if config.get('key') and config.get('cert'): + config['cert'] = (config['cert'], config['key']) + + kwargs = {p: config.get(p) for p in ('host', 'port', 'token', 'scheme', 'cert', 'ca_cert') if config.get(p)} + + verify = config.get('verify') + if not isinstance(verify, bool): + verify = parse_bool(verify) + if isinstance(verify, bool): + kwargs['verify'] = verify + + self._client = ConsulClient(**kwargs) self.set_retry_timeout(config['retry_timeout']) self.set_ttl(config.get('ttl') or 30) self._last_session_refresh = 0 diff --git a/requirements.txt b/requirements.txt index 5aeb8568..a66aaef5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ requests six >= 1.7 kazoo==2.2.1 python-etcd>=0.4.3,<0.5 -python-consul==0.7.0 +python-consul>=0.7.0 click>=4.1 prettytable>=0.7 tzlocal diff --git a/tests/test_consul.py b/tests/test_consul.py index 04700b3d..004b6fc0 100644 --- a/tests/test_consul.py +++ b/tests/test_consul.py @@ -69,6 +69,10 @@ class TestConsul(unittest.TestCase): @patch.object(consul.Consul.KV, 'get', kv_get) @patch.object(consul.Consul.KV, 'delete', Mock()) def setUp(self): + Consul({'ttl': 30, 'scope': 't', 'name': 'p', 'url': 'https://l:1', 'retry_timeout': 10, + 'verify': 'on', 'key': 'foo', 'cert': 'bar', 'cacert': 'buz'}) + Consul({'ttl': 30, 'scope': 't', 'name': 'p', 'url': 'https://l:1', 'retry_timeout': 10, + 'verify': 'on', 'cert': 'bar', 'cacert': 'buz'}) self.c = Consul({'ttl': 30, 'scope': 'test', 'name': 'postgresql1', 'host': 'localhost:1', 'retry_timeout': 10}) self.c._base_path = '/service/good' self.c._load_cluster()