From 2f84e9f4ec38bec637b86b5b229d52c163658346 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Tue, 23 Feb 2016 16:44:09 +0100 Subject: [PATCH 1/5] Add support for ZooKeeper/Exhibitor DCS URI in patronictl ... -d --- patroni/ctl.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/patroni/ctl.py b/patroni/ctl.py index fb4c241b..48a0c413 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -16,6 +16,7 @@ from six.moves.urllib_parse import urlparse import logging from .etcd import Etcd +from .zookeeper import ZooKeeper from .exceptions import PatroniCtlException from .postgresql import parseurl @@ -44,12 +45,12 @@ def parse_dcs(dcs): parsed = urlparse('//' + dcs) if scheme == '': - default_schemes = {'2181': 'zookeeper', '8500': 'consul'} + default_schemes = {'2181': 'zookeeper', '8181': 'exhibitor', '8500': 'consul'} scheme = default_schemes.get(str(parsed.port), 'etcd') port = parsed.port if port is None: - default_ports = {'consul': 8500, 'zookeeper': 2181} + default_ports = {'consul': 8500, 'zookeeper': 2181, 'exhibitor': 8181} port = default_ports.get(str(scheme), 4001) return {'scheme': str(scheme), 'hostname': str(parsed.hostname), 'port': int(port)} @@ -103,6 +104,12 @@ def get_dcs(config, scope): if scheme == 'etcd': return Etcd(name=scope, config={'scope': scope, 'host': '{0}:{1}'.format(hostname, port)}) + if scheme == 'zookeeper': + return ZooKeeper(name=scope, config={'scope': scope, 'hosts': [hostname], 'port': port}) + + if scheme == 'exhibitor': + return ZooKeeper(name=scope, config={'scope': scope, 'exhibitor': {'hosts': [hostname], 'port': port}}) + raise PatroniCtlException('Can not find suitable configuration of distributed configuration store') From a1417875a2e2e223f3b8a309c4e19047db644a93 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Tue, 23 Feb 2016 17:04:16 +0100 Subject: [PATCH 2/5] Add dummy patronictl tests with ZooKeeper --- tests/test_ctl.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_ctl.py b/tests/test_ctl.py index fb63aebd..e0e5fc99 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -16,10 +16,12 @@ from patroni.ctl import ctl, members, store_config, load_config, output_members, wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure from patroni.ha import Ha from patroni.etcd import Etcd, Client +from patroni.zookeeper import ZooKeeper from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \ get_cluster_initialized_with_only_leader, MockPostgresql, MockPatroni, run_async, \ get_cluster_not_initialized_without_leader from test_etcd import etcd_read, etcd_write, requests_get, socket_getaddrinfo, MockResponse +from test_zookeeper import MockKazooClient from test_postgresql import MockConnect, psycopg2_connect CONFIG_FILE_PATH = './test-ctl.yaml' @@ -52,6 +54,7 @@ def test_rw_config(): class TestCtl(unittest.TestCase): @patch('socket.getaddrinfo', socket_getaddrinfo) + @patch('patroni.zookeeper.KazooClient', MockKazooClient) def setUp(self): self.runner = CliRunner() with patch.object(Client, 'machines') as mock_machines: @@ -61,6 +64,7 @@ class TestCtl(unittest.TestCase): self.e.client.read = etcd_read self.e.client.write = etcd_write self.e.client.delete = Mock(side_effect=etcd.EtcdException()) + self.zk = ZooKeeper('foo', {'ttl': 30, 'hosts': ['ok:2181'], 'scope': 'test'}) self.ha = Ha(MockPatroni(self.p, self.e)) self.ha._async_executor.run_async = run_async self.ha.old_cluster = self.e.get_cluster() @@ -377,3 +381,13 @@ leader''') ]) assert result.exit_code == 0 + + @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'zookeeper', 'hostname': 'localhost', 'port': 2181}})) + def test_zookeeper(self): + with patch('patroni.ctl.get_dcs', Mock(return_value=self.zk)): + self.runner.invoke(ctl, ['list']) + + @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'exhibitor', 'hostname': 'localhost', 'port': 8181}})) + def test_exhibitor(self): + with patch('patroni.ctl.get_dcs', Mock(return_value=self.zk)): + self.runner.invoke(ctl, ['list']) From 524cfafbbe07a1fe6a471245e5255edf803be7b5 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Tue, 23 Feb 2016 17:18:16 +0100 Subject: [PATCH 3/5] Don't mock get_dcs() for ZK, we are trying to test it --- tests/test_ctl.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_ctl.py b/tests/test_ctl.py index 55612a94..f8f2783a 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -390,10 +390,8 @@ leader''') @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'zookeeper', 'hostname': 'localhost', 'port': 2181}})) def test_zookeeper(self): - with patch('patroni.ctl.get_dcs', Mock(return_value=self.zk)): - self.runner.invoke(ctl, ['list']) + self.runner.invoke(ctl, ['list']) @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'exhibitor', 'hostname': 'localhost', 'port': 8181}})) def test_exhibitor(self): - with patch('patroni.ctl.get_dcs', Mock(return_value=self.zk)): - self.runner.invoke(ctl, ['list']) + self.runner.invoke(ctl, ['list']) From 16b321e0a53494d70e5c606f023d78aef02c9d94 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Tue, 23 Feb 2016 17:33:15 +0100 Subject: [PATCH 4/5] Add dummy cluster name to test_ctl / zookeeper --- tests/test_ctl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_ctl.py b/tests/test_ctl.py index f8f2783a..4092a669 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -390,8 +390,8 @@ leader''') @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'zookeeper', 'hostname': 'localhost', 'port': 2181}})) def test_zookeeper(self): - self.runner.invoke(ctl, ['list']) + self.runner.invoke(ctl, ['list', 'foo']) @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'exhibitor', 'hostname': 'localhost', 'port': 8181}})) def test_exhibitor(self): - self.runner.invoke(ctl, ['list']) + self.runner.invoke(ctl, ['list', 'foo']) From 9c12eb671da4c83e7f3f2dae35bb2eed2b644741 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 24 Feb 2016 12:10:58 +0100 Subject: [PATCH 5/5] Fix unit tests --- tests/test_ctl.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/tests/test_ctl.py b/tests/test_ctl.py index 4092a669..ed3aa1b9 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -9,7 +9,6 @@ from mock import patch, Mock, MagicMock from patroni.ctl import ctl, members, store_config, load_config, output_members, post_patroni, get_dcs, \ wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure from patroni.etcd import Etcd, Client -from patroni.zookeeper import ZooKeeper from patroni.exceptions import PatroniCtlException from psycopg2 import OperationalError from test_etcd import etcd_read, etcd_write, requests_get, socket_getaddrinfo, MockResponse @@ -48,7 +47,6 @@ def test_rw_config(): class TestCtl(unittest.TestCase): @patch('socket.getaddrinfo', socket_getaddrinfo) - @patch('patroni.zookeeper.KazooClient', MockKazooClient) def setUp(self): self.runner = CliRunner() with patch.object(Client, 'machines') as mock_machines: @@ -57,7 +55,6 @@ class TestCtl(unittest.TestCase): self.e.client.read = etcd_read self.e.client.write = etcd_write self.e.client.delete = Mock(side_effect=EtcdException) - self.zk = ZooKeeper('foo', {'ttl': 30, 'hosts': ['ok:2181'], 'scope': 'test'}) @patch('psycopg2.connect', psycopg2_connect) def test_get_cursor(self): @@ -177,7 +174,11 @@ other y''') assert 'Failover failed' in result.output - def test_(self): + @patch('patroni.zookeeper.KazooClient', MockKazooClient) + @patch('requests.get', requests_get) + def test_get_dcs(self): + self.assertIsNotNone(get_dcs({'dcs': {'scheme': 'zookeeper', 'hostname': 'foo', 'port': 2181}}, 'dummy')) + self.assertIsNotNone(get_dcs({'dcs': {'scheme': 'exhibitor', 'hostname': 'exhibitor', 'port': 8181}}, 'dummy')) self.assertRaises(PatroniCtlException, get_dcs, {'scheme': 'dummy'}, 'dummy') @patch('psycopg2.connect', psycopg2_connect) @@ -387,11 +388,3 @@ leader''') ]) assert result.exit_code == 0 - - @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'zookeeper', 'hostname': 'localhost', 'port': 2181}})) - def test_zookeeper(self): - self.runner.invoke(ctl, ['list', 'foo']) - - @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'exhibitor', 'hostname': 'localhost', 'port': 8181}})) - def test_exhibitor(self): - self.runner.invoke(ctl, ['list', 'foo'])