From b5a5ea2a7500e563c38dcdf0c3c8154becbef5ab Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 9 Sep 2015 12:15:52 +0200 Subject: [PATCH 1/6] Add SSL wrapper around restapi server socket If config['restapi']['certfile'] is specified and not empty http server would be wrapped into SSL and api connection string changed accordingly: http:// => https:// --- patroni/api.py | 14 +++++++++++++- tests/test_patroni.py | 10 ++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/patroni/api.py b/patroni/api.py index d9a0f527..979b6cbb 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -52,11 +52,23 @@ class RestApiHandler(BaseHTTPRequestHandler): class RestApiServer(ThreadingMixIn, HTTPServer, Thread): def __init__(self, patroni, config): - self.connection_string = 'http://{}/patroni'.format(config.get('connect_address', None) or config['listen']) host, port = config['listen'].split(':') HTTPServer.__init__(self, (host, int(port)), RestApiHandler) Thread.__init__(self, target=self.serve_forever) self._set_fd_cloexec(self.socket) + + protocol = 'http' + + # wrap socket with ssl if 'certfile' is defined in a config.yaml + # Sometime it's also needed to pass reference to a 'keyfile'. + options = {option: config[option] for option in ['certfile', 'keyfile'] if option in config} + if options.get('certfile', None): + import ssl + self.socket = ssl.wrap_socket(self.socket, server_side=True, **options) + protocol = 'https' + + self.connection_string = '{}://{}/patroni'.format(protocol, config.get('connect_address', config['listen'])) + self.patroni = patroni self.daemon = True diff --git a/tests/test_patroni.py b/tests/test_patroni.py index a45ebf62..68af3866 100644 --- a/tests/test_patroni.py +++ b/tests/test_patroni.py @@ -2,6 +2,7 @@ import datetime import patroni.zookeeper import psycopg2 import subprocess +import ssl import sys import time import unittest @@ -32,6 +33,10 @@ def time_sleep(*args): raise SleepException() +def ssl_wrap_socket(socket, *args, **kwargs): + return socket + + class Mock_BaseServer__is_shut_down: def set(self): @@ -62,8 +67,10 @@ class TestPatroni(unittest.TestCase): RestApiServer._BaseServer__is_shut_down = Mock_BaseServer__is_shut_down() RestApiServer._BaseServer__shutdown_request = True RestApiServer.socket = 0 + ssl.wrap_socket = ssl_wrap_socket with open('postgres0.yml', 'r') as f: config = yaml.load(f) + config['restapi']['certfile'] = 'dump' with patch.object(Client, 'machines') as mock_machines: mock_machines.__get__ = Mock(return_value=['http://remotehost:2379']) self.p = Patroni(config) @@ -147,3 +154,6 @@ class TestPatroni(unittest.TestCase): def test_schedule_next_run(self): self.p.next_run = time.time() - self.p.nap_time - 1 self.p.schedule_next_run() + + def test_api_connection_string(self): + self.assertTrue(self.p.api.connection_string.startswith('https://')) From 1d9333dcdccf917e0375af6432612c274aaf0340 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 9 Sep 2015 12:19:50 +0200 Subject: [PATCH 2/6] Update documentation and configs with examples of usage of SSL. --- README.rst | 3 +++ postgres0.yml | 2 ++ postgres1.yml | 2 ++ 3 files changed, 7 insertions(+) diff --git a/README.rst b/README.rst index 3e19f0a2..21629de9 100644 --- a/README.rst +++ b/README.rst @@ -61,6 +61,9 @@ settings: health-check information for haproxy. - *connect\_address*: ip address + port through which restapi is accessible. +- *certfile*: Specifies a file with the certificate in the PEM format. + If certfile is not specified or empty API server will work without SSL. +- *keyfile*: Specifies a file with the secret key in the PEM format. - *etcd* - *scope*: the relative path used on etcd's http api for this diff --git a/postgres0.yml b/postgres0.yml index 659a4db2..7c086d7d 100644 --- a/postgres0.yml +++ b/postgres0.yml @@ -4,6 +4,8 @@ scope: &scope batman restapi: listen: 127.0.0.1:8008 connect_address: 127.0.0.1:8008 +# certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem +# keyfile: /etc/ssl/private/ssl-cert-snakeoil.key etcd: scope: *scope ttl: *ttl diff --git a/postgres1.yml b/postgres1.yml index bc8b6fd1..988a74f6 100644 --- a/postgres1.yml +++ b/postgres1.yml @@ -4,6 +4,8 @@ scope: &scope batman restapi: listen: 127.0.0.1:8009 connect_address: 127.0.0.1:8009 +# certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem +# keyfile: /etc/ssl/private/ssl-cert-snakeoil.key etcd: scope: *scope ttl: *ttl From c409ee4d371c9d101e0220e7751b0acb281feb83 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 9 Sep 2015 12:39:48 +0200 Subject: [PATCH 3/6] Fix formatting in a README.rst --- README.rst | 126 +++++++++++++++++++---------------------------------- 1 file changed, 45 insertions(+), 81 deletions(-) diff --git a/README.rst b/README.rst index 21629de9..b73185e4 100644 --- a/README.rst +++ b/README.rst @@ -52,97 +52,61 @@ YAML Configuration For an example file, see ``postgres0.yml``. Below is an explanation of settings: -- *ttl*: the TTL to acquire the leader lock. Think of it as the length - of time before automatic failover process is initiated. +- *ttl*: the TTL to acquire the leader lock. Think of it as the length of time before automatic failover process is initiated. - *loop\_wait*: the number of seconds the loop will sleep -- *restapi* -- *listen*: ip address + port that Patroni will listen to provide - health-check information for haproxy. -- *connect\_address*: ip address + port through which restapi is - accessible. -- *certfile*: Specifies a file with the certificate in the PEM format. - If certfile is not specified or empty API server will work without SSL. -- *keyfile*: Specifies a file with the secret key in the PEM format. +- *restapi*: + - *listen*: ip address + port that Patroni will listen to provide health-check information for haproxy. + - *connect\_address*: ip address + port through which restapi is accessible. + - *certfile*: (optional) Specifies a file with the certificate in the PEM format. If certfile is not specified or empty API server will work without SSL. + - *keyfile*: (optional) Specifies a file with the secret key in the PEM format. -- *etcd* -- *scope*: the relative path used on etcd's http api for this - deployment, thus you can run multiple HA deployments from a single - etcd -- *ttl*: the TTL to acquire the leader lock. Think of it as the length - of time before automatic failover process is initiated. -- *host*: the host:port for the etcd endpoint +- *etcd*: + - *scope*: the relative path used on etcd's http api for this deployment, thus you can run multiple HA deployments from a single etcd + - *ttl*: the TTL to acquire the leader lock. Think of it as the length of time before automatic failover process is initiated. + - *host*: the host:port for the etcd endpoint -- *zookeeper* -- *scope*: the relative path used on etcd's http api for this - deployment, thus you can run multiple HA deployments from a single - etcd -- *session\_timeout*: the TTL to acquire the leader lock. Think of it - as the length of time before automatic failover process is initiated. -- *reconnect\_timeout*: how long we should try to reconnect to - ZooKeeper after connection loss. After this timeout we assume that we - don't have lock anymore and will restart in read-only mode. -- *hosts*: list of ZooKeeper cluster members in format: [ - 'host1:port1', 'host2:port2', 'etc...'] -- *exhibitor*: if you are running ZooKeeper cluster under Exhibitor - supervisory the following section could be interesting for you +- *zookeeper*: + - *scope*: the relative path used on etcd's http api for this deployment, thus you can run multiple HA deployments from a single etcd + - *session\_timeout*: the TTL to acquire the leader lock. Think of it as the length of time before automatic failover process is initiated. + - *reconnect\_timeout*: how long we should try to reconnect to ZooKeeper after connection loss. After this timeout we assume that we don't have lock anymore and will restart in read-only mode. + - *hosts*: list of ZooKeeper cluster members in format: ['host1:port1', 'host2:port2', 'etc...'] + - *exhibitor*: if you are running ZooKeeper cluster under Exhibitor supervisory the following section could be interesting for you + - *poll\_interval*: how often list of ZooKeeper and Exhibitor nodes should be updated from Exhibitor + - *port*: Exhibitor port + - *hosts*: initial list of Exhibitor (ZooKeeper) nodes in format: ['host1', 'host2', 'etc...' ]. This list would be updated automatically when Exhibitor (ZooKeeper) cluster topology changes. - - *poll\_interval*: how often list of ZooKeeper and Exhibitor nodes - should be updated from Exhibitor - - *port*: Exhibitor port - - *hosts*: initial list of Exhibitor (ZooKeeper) nodes in format: [ - 'host1', 'host2', 'etc...' ]. This list would be updated - automatically when Exhibitor (ZooKeeper) cluster topology changes. +- *postgresql*: + - *name*: the name of the Postgres host, must be unique for the cluster + - *listen*: ip address + port that Postgres listening. Must be accessible from other nodes in the cluster if using streaming replication. + - *connect\_address*: ip address + port through which Postgres is accessible from other nodes and applications. + - *data\_dir*: file path to initialize and store Postgres data files + - *maximum\_lag\_on\_failover*: the maximum bytes a follower may lag + - *use\_slots*: whether or not to use replication_slots. Must be False for PostgreSQL 9.3, and you should comment out max_replication_slots. before it is not eligible become leader + - *pg\_hba*: list of lines which should be added to pg\_hba.conf + - *- host all all 0.0.0.0/0 md5* -- *postgresql* -- *name*: the name of the Postgres host, must be unique for the cluster -- *listen*: ip address + port that Postgres listening. Must be - accessible from other nodes in the cluster if using streaming - replication. -- *connect\_address*: ip address + port through which Postgres is - accessible from other nodes and applications. -- *data\_dir*: file path to initialize and store Postgres data files -- *maximum\_lag\_on\_failover*: the maximum bytes a follower may lag -- *use\_slots*: whether or not to use replication_slots. Must be False for PostgreSQL 9.3, and you should comment out max_replication_slots. - before it is not eligible become leader -- *pg\_hba*: list of lines which should be added to pg\_hba.conf + - *replication*: + - *username*: replication username, user will be created during initialization + - *password*: replication password, user will be created during initialization + - *network*: network setting for replication in pg\_hba.conf - - *- host all all 0.0.0.0/0 md5* + - *callbacks* callback scripts to run on certain actions. Patroni will pass current action, role and cluster name. See scripts/aws.py as an example on how to write them. + - *on\_start*: a script to run when the cluster starts + - *on\_stop*: a script to run when the cluster stops + - *on\_restart*: a script to run when the cluster restarts + - *on\_reload*: a script to run when configuration reload is triggered + - *on\_role\_change*: a script to run when the cluster is being promoted or demoted -- *replication* + - *superuser*: + - *password*: password for postgres user. It would be set during initialization - - *username*: replication username, user will be created during - initialization - - *password*: replication password, user will be created during - initialization - - *network*: network setting for replication in pg\_hba.conf + - *admin*: + - *username*: admin username, user will be created during initialization. It would have CREATEDB and REATEROLE privileges + - *password*: admin password, user will be created during initialization. -- *callbacks* callback scripts to run on certain actions. Patroni will - pass current action, role and cluster name. See scripts/aws.py as an - example on how to write them. - - - *on\_start*: a script to run when the cluster starts - - *on\_stop*: a script to run when the cluster stops - - *on\_restart*: a script to run when the cluster restarts - - *on\_reload*: a script to run when configuration reload is - triggered - - *on\_role\_change*: a script to run when the cluster is being - promoted or demoted - -- *superuser* - - - *password*: password for postgres user. It would be set during - initialization - -- *admin*: - - - *username*: admin username, user will be created during - initialization. It would have CREATEDB and CREATEROLE privileges - - *password*: admin password, user will be created during - initialization. - -- *recovery\_conf*: additional configuration settings written to recovery.conf when configuring follower -- *parameters*: list of configuration settings for Postgres. Many of these are required for replication to work. + - *recovery\_conf*: additional configuration settings written to recovery.conf when configuring follower + - *parameters*: list of configuration settings for Postgres. Many of these are required for replication to work. Replication choices ------------------- From 5a99faf96786ba8376ec9fa57d774d6cf2f8cd4e Mon Sep 17 00:00:00 2001 From: Feike Steenbergen Date: Wed, 9 Sep 2015 12:55:52 +0200 Subject: [PATCH 4/6] Update README.rst Typo --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index b73185e4..67f8a1f9 100644 --- a/README.rst +++ b/README.rst @@ -102,7 +102,7 @@ settings: - *password*: password for postgres user. It would be set during initialization - *admin*: - - *username*: admin username, user will be created during initialization. It would have CREATEDB and REATEROLE privileges + - *username*: admin username, user will be created during initialization. It would have CREATEDB and CREATEROLE privileges - *password*: admin password, user will be created during initialization. - *recovery\_conf*: additional configuration settings written to recovery.conf when configuring follower From abcaf2b94ab7be90db93934f154e5ea9fa398fa7 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 10 Sep 2015 15:25:08 +0200 Subject: [PATCH 5/6] Possibility to protect some endpoints with basic-auth user:passwd pair should be configured in restapi section of main configuration file in following format: restapi: auth: 'username:password' Plus implemented some simple routing mechanisms: GET /foo => do_GET_foo() POST /bar => do_POST_bar() --- patroni/api.py | 64 +++++++++++++++++++++++++++++++++++++++++++ postgres0.yml | 1 + postgres1.yml | 1 + tests/test_api.py | 28 ++++++++++++++++++- tests/test_patroni.py | 10 +------ 5 files changed, 94 insertions(+), 10 deletions(-) diff --git a/patroni/api.py b/patroni/api.py index d9a0f527..b99f2cb8 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -1,3 +1,4 @@ +import base64 import fcntl import json import logging @@ -10,9 +11,36 @@ from threading import Thread logger = logging.getLogger(__name__) +def check_auth(func): + """Decorator function to check authorization header. + + Usage example: + @check_auth + def do_PUT_foo(): + pass + """ + def wrapper(handler): + if handler.check_auth_header(): + return func(handler) + return wrapper + + class RestApiHandler(BaseHTTPRequestHandler): + def send_auth_request(self, body): + self.send_response(401) + self.send_header('WWW-Authenticate', 'Basic realm=\"Patroni\"') + self.send_header('Content-type', 'text/html') + self.end_headers() + + def check_auth_header(self): + auth_header = self.headers.get('Authorization') + status = self.server.check_auth_header(auth_header) + return not status or self.send_auth_request(status) + def do_GET(self): + """Default method for processing all GET requests which can not be routed to other methods""" + response = self.get_postgresql_status() path = '/master' if self.path == '/' else self.path @@ -23,6 +51,31 @@ class RestApiHandler(BaseHTTPRequestHandler): self.end_headers() self.wfile.write(json.dumps(response).encode('utf-8')) + @check_auth + def do_GET_sampleauth(self): + self.send_response(200) + self.send_header('Content-Type', 'text/html') + self.end_headers() + self.wfile.write(b'Hello!') + + def parse_request(self): + """Override parse_request method to enrich basic functionality of `BaseHTTPRequestHandler` class + + Original class can only invoke do_GET, do_POST, do_PUT, etc method implementations if they are defined. + But we would like to have at least some simple routing mechanism, i.e.: + GET /uri1/part2 request should invoke `do_GET_uri1()` + POST /other should invoke `do_POST_other()` + + If the `do__` method does not exists we'll fallback to original behavior.""" + + ret = BaseHTTPRequestHandler.parse_request(self) + if ret: + mname = self.path.lstrip('/').split('/')[0] + mname = self.command + ('_' + mname if mname else '') + if hasattr(self, 'do_' + mname): + self.command = mname + return ret + def get_postgresql_status(self): try: row = self.server.query("""SELECT to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'), @@ -52,6 +105,7 @@ class RestApiHandler(BaseHTTPRequestHandler): class RestApiServer(ThreadingMixIn, HTTPServer, Thread): def __init__(self, patroni, config): + self._auth_key = base64.b64encode(config['auth'].encode('utf-8')).decode('utf-8') if 'auth' in config else None self.connection_string = 'http://{}/patroni'.format(config.get('connect_address', None) or config['listen']) host, port = config['listen'].split(':') HTTPServer.__init__(self, (host, int(port)), RestApiHandler) @@ -71,3 +125,13 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread): def _set_fd_cloexec(fd): flags = fcntl.fcntl(fd, fcntl.F_GETFD) fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) + + def check_basic_auth_key(self, key): + return self._auth_key == key + + def check_auth_header(self, auth_header): + if self._auth_key: + if auth_header is None: + return 'no auth header received' + if not auth_header.startswith('Basic ') or not self.check_basic_auth_key(auth_header[6:]): + return 'not authenticated' diff --git a/postgres0.yml b/postgres0.yml index 659a4db2..eb9d10ec 100644 --- a/postgres0.yml +++ b/postgres0.yml @@ -4,6 +4,7 @@ scope: &scope batman restapi: listen: 127.0.0.1:8008 connect_address: 127.0.0.1:8008 + auth: 'username:password' etcd: scope: *scope ttl: *ttl diff --git a/postgres1.yml b/postgres1.yml index bc8b6fd1..52a0bc26 100644 --- a/postgres1.yml +++ b/postgres1.yml @@ -4,6 +4,7 @@ scope: &scope batman restapi: listen: 127.0.0.1:8009 connect_address: 127.0.0.1:8009 + auth: 'username:password' etcd: scope: *scope ttl: *ttl diff --git a/tests/test_api.py b/tests/test_api.py index aecf3df6..46f9e5ae 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -3,13 +3,27 @@ import unittest from patroni.api import RestApiHandler, RestApiServer from six import BytesIO as IO +from six.moves import BaseHTTPServer from test_postgresql import psycopg2_connect +def nop(*args, **kwargs): + pass + + def throws(*args, **kwargs): raise psycopg2.OperationalError() +class Mock_BaseServer__is_shut_down: + + def set(self): + pass + + def clear(self): + pass + + class MockPostgresql: def connection(self): @@ -37,7 +51,7 @@ class MockRequest: class MockRestApiServer(RestApiServer): def __init__(self, Handler, path, *args): - self.patroni = MockPatroni() + super(MockRestApiServer, self).__init__(MockPatroni(), {'listen': '127.0.0.1:8008', 'auth': 'test:test'}) if len(args) > 0: self.query = args[0] Handler(MockRequest(path), ('0.0.0.0', 8080), self) @@ -46,8 +60,20 @@ class MockRestApiServer(RestApiServer): class TestRestApiHandler(unittest.TestCase): def __init__(self, method_name='runTest'): + self.setUp = self.set_up super(TestRestApiHandler, self).__init__(method_name) + def set_up(self): + BaseHTTPServer.HTTPServer.__init__ = nop + RestApiServer._BaseServer__is_shut_down = Mock_BaseServer__is_shut_down() + RestApiServer._BaseServer__shutdown_request = True + RestApiServer.socket = 0 + def test_do_GET(self): MockRestApiServer(RestApiHandler, b'GET /') MockRestApiServer(RestApiHandler, b'GET /', throws) + + def test_do_GET_sampleauth(self): + MockRestApiServer(RestApiHandler, b'GET /sampleauth') + MockRestApiServer(RestApiHandler, b'GET /sampleauth\nAuthorization:') + MockRestApiServer(RestApiHandler, b'GET /sampleauth\nAuthorization: Basic dGVzdDp0ZXN0') diff --git a/tests/test_patroni.py b/tests/test_patroni.py index a45ebf62..285d0e95 100644 --- a/tests/test_patroni.py +++ b/tests/test_patroni.py @@ -14,6 +14,7 @@ from patroni.etcd import Etcd from patroni import Patroni, main from patroni.zookeeper import ZooKeeper from six.moves import BaseHTTPServer +from test_api import Mock_BaseServer__is_shut_down from test_etcd import Client, etcd_read, etcd_write from test_ha import true, false from test_postgresql import Postgresql, subprocess_call, psycopg2_connect @@ -32,15 +33,6 @@ def time_sleep(*args): raise SleepException() -class Mock_BaseServer__is_shut_down: - - def set(self): - pass - - def clear(self): - pass - - class TestPatroni(unittest.TestCase): def __init__(self, method_name='runTest'): From 3d7c6118de7fc12fbb01255bb6c755e326a025b6 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 10 Sep 2015 16:29:17 +0200 Subject: [PATCH 6/6] Add missing body to authenticate request --- patroni/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/patroni/api.py b/patroni/api.py index b99f2cb8..02db2900 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -32,6 +32,7 @@ class RestApiHandler(BaseHTTPRequestHandler): self.send_header('WWW-Authenticate', 'Basic realm=\"Patroni\"') self.send_header('Content-type', 'text/html') self.end_headers() + self.wfile.write(body.encode('utf-8')) def check_auth_header(self): auth_header = self.headers.get('Authorization')