Merge branch 'etcd-cluster-support' of github.com:CyberDem0n/governor into etcd-cluster-support

Conflicts:
	requirements-py2.txt
	tests/test_governor.py

tests/test_etcd.py

@@ -1,10 +1,13 @@
-import unittest
-import requests
-import time
+import dns.resolver
 import json
+import requests
+import socket
+import time
+import unittest
 
-from helpers.etcd import Cluster, Etcd
-from helpers.errors import EtcdError, CurrentLeaderError
+from dns.exception import DNSException
+from helpers.errors import EtcdError, CurrentLeaderError, EtcdConnectionFailed
+from helpers.etcd import Client, Cluster, Etcd
 
 
 class MockResponse:
@@ -26,10 +29,19 @@ class MockPostgresql:
 
 
 def requests_get(url, **kwargs):
-    if url.startswith('http://local'):
-        raise requests.exceptions.RequestException()
+    members = '[{"id":14855829450254237642,"peerURLs":["http://localhost:2380","http://localhost:7001"],"name":"default","clientURLs":["http://localhost:2379","http://localhost:4001"]}]'
     response = MockResponse()
-    if url.startswith('http://remote') or url.startswith('http://127.0.0.1'):
+    if url.endswith('/v2/members'):
+        response.content = '{"members": ' + members + '}'
+        if url.startswith('http://error'):
+            response.status_code = 404
+    elif url.endswith('/members'):
+        response.content = members
+    elif url.endswith('/bad_response'):
+        response.content = '{'
+    elif url.startswith('http://local'):
+        raise requests.exceptions.RequestException()
+    elif url.startswith('http://remote') or url.startswith('http://127.0.0.1') or url.startswith('http://error'):
         response.content = '{"action":"get","node":{"key":"/service/batman5","dir":true,"nodes":[{"key":"/service/batman5/initialize","value":"postgresql0","modifiedIndex":1582,"createdIndex":1582},{"key":"/service/batman5/leader","value":"postgresql1","expiration":"2015-05-15T09:11:00.037397538Z","ttl":21,"modifiedIndex":20728,"createdIndex":20434},{"key":"/service/batman5/optime","dir":true,"nodes":[{"key":"/service/batman5/optime/leader","value":"2164261704","modifiedIndex":20729,"createdIndex":20729}],"modifiedIndex":20437,"createdIndex":20437},{"key":"/service/batman5/members","dir":true,"nodes":[{"key":"/service/batman5/members/postgresql1","value":"postgres://replicator:[email protected]:5434/postgres?application_name=http://127.0.0.1:8009/governor","expiration":"2015-05-15T09:10:59.949384522Z","ttl":21,"modifiedIndex":20727,"createdIndex":20727},{"key":"/service/batman5/members/postgresql0","value":"postgres://replicator:[email protected]:5433/postgres?application_name=http://127.0.0.1:8008/governor","expiration":"2015-05-15T09:11:09.611860899Z","ttl":30,"modifiedIndex":20730,"createdIndex":20730}],"modifiedIndex":1581,"createdIndex":1581}],"modifiedIndex":1581,"createdIndex":1581}}'
     elif url.startswith('http://other'):
         response.status_code = 404
@@ -51,7 +63,7 @@ def requests_put(url, **kwargs):
     return response
 
 
-def requests_delete(url):
+def requests_delete(url, **kwargs):
     if url.startswith('http://local'):
         raise requests.exceptions.RequestException()
     response = MockResponse()
@@ -63,6 +75,81 @@ def time_sleep(_):
     pass
 
 
+def time_sleep_exception(_):
+    raise Exception()
+
+
+class MockSRV:
+    port = 2380
+    target = '127.0.0.1'
+
+
+def dns_query(name, type):
+    if name == '_etcd-server._tcp.blabla':
+        return []
+    elif name == '_etcd-server._tcp.exception':
+        raise DNSException()
+    return [MockSRV()]
+
+
+def socket_getaddrinfo(*args):
+    if args[0] == 'ok':
+        return [(2, 1, 6, '', ('127.0.0.1', 2379)), (2, 1, 6, '', ('127.0.0.1', 2379))]
+    raise socket.error()
+
+
+class TestClient(unittest.TestCase):
+
+    def __init__(self, method_name='runTest'):
+        self.setUp = self.set_up
+        super(TestClient, self).__init__(method_name)
+
+    def set_up(self):
+        socket.getaddrinfo = socket_getaddrinfo
+        requests.get = requests_get
+        requests.put = requests_put
+        requests.delete = requests_delete
+        dns.resolver.query = dns_query
+        self.client = Client({'discovery_srv': 'test'})
+
+    def test__get(self):
+        self.assertRaises(EtcdError, self.client._get, '/bad_response')
+
+    def test_get_srv_record(self):
+        self.assertEquals(Client.get_srv_record('blabla'), [])
+        self.assertEquals(Client.get_srv_record('exception'), [])
+
+    def test_get_client_urls_from_dns(self):
+        self.assertEquals(Client.get_client_urls_from_dns('ok:2379'), ['http://127.0.0.1:2379/v2'])
+
+    def test_load_members(self):
+        self.client._base_uri = self.client._base_uri.replace('localhost', 'error_code')
+        self.assertRaises(EtcdError, self.client.load_members)
+        self.client._base_uri = None
+        self.client._config = {}
+        self.assertRaises(Exception, self.client.load_members)
+
+    def test_get(self):
+        self.client._base_uri = None
+        self.assertRaises(EtcdConnectionFailed, self.client.get, '')
+        self.client._members_cache = ['http://error_code:4001/v2']
+        self.client.get('')
+
+    def test_put(self):
+        self.client._base_uri = None
+        self.assertRaises(EtcdConnectionFailed, self.client.put, '')
+        self.client._base_uri = 'http://localhost:4001/v2'
+        self.client._members_cache = ['http://error_code:4001/v2']
+        self.client.put('')
+
+    def test_delete(self):
+        self.client._base_uri = None
+        self.assertRaises(EtcdConnectionFailed, self.client.delete, '')
+        self.client._base_uri = 'http://localhost:4001/v2'
+        self.client._members_cache = ['http://error_code:4001/v2']
+        self.client.delete('')
+
+
 class TestEtcd(unittest.TestCase):
 
     def __init__(self, method_name='runTest'):
@@ -70,11 +157,16 @@ class TestEtcd(unittest.TestCase):
         super(TestEtcd, self).__init__(method_name)
 
     def set_up(self):
+        socket.getaddrinfo = socket_getaddrinfo
         requests.get = requests_get
         requests.put = requests_put
         requests.delete = requests_delete
         time.sleep = time_sleep
-        self.etcd = Etcd({'ttl': 30, 'host': 'localhost', 'scope': 'test'})
+        self.etcd = Etcd({'ttl': 30, 'host': 'localhost:2379', 'scope': 'test'})
+
+    def test_get_etcd_client(self):
+        time.sleep = time_sleep_exception
+        self.assertRaises(Exception, self.etcd.get_etcd_client, {'host': 'error:2379'})
 
     def test_get_client_path(self):
         self.assertRaises(Exception, self.etcd.get_client_path, '', 2)
@@ -87,12 +179,12 @@ class TestEtcd(unittest.TestCase):
 
     def test_get_cluster(self):
         self.assertRaises(EtcdError, self.etcd.get_cluster)
-        self.etcd.base_client_url = self.etcd.base_client_url.replace('local', 'remote')
+        self.etcd.client._base_uri = self.etcd.client._base_uri.replace('local', 'remote')
         cluster = self.etcd.get_cluster()
         self.assertIsInstance(cluster, Cluster)
-        self.etcd.base_client_url = self.etcd.base_client_url.replace('remote', 'other')
+        self.etcd.client._base_uri = self.etcd.client._base_uri.replace('remote', 'other')
         self.etcd.get_cluster()
-        self.etcd.base_client_url = self.etcd.base_client_url.replace('other', 'noleader')
+        self.etcd.client._base_uri = self.etcd.client._base_uri.replace('other', 'noleader')
         self.etcd.get_cluster()
 
     def test_current_leader(self):
@@ -108,9 +200,9 @@ class TestEtcd(unittest.TestCase):
         self.assertFalse(self.etcd.attempt_to_acquire_leader(''))
 
     def test_update_leader(self):
-        self.etcd.base_client_url = self.etcd.base_client_url.replace('local', 'remote')
+        url = self.etcd.client._base_uri = self.etcd.client._base_uri.replace('local', 'remote')
         self.assertTrue(self.etcd.update_leader(MockPostgresql()))
-        self.etcd.base_client_url = self.etcd.base_client_url.replace('remote', 'other')
+        self.etcd.client._base_uri = url.replace('remote', 'other')
         self.assertFalse(self.etcd.update_leader(MockPostgresql()))
 
     def test_race(self):

tests/test_governor.py

@@ -69,8 +69,7 @@ class TestGovernor(unittest.TestCase):
 
     def test_governor_initialize(self):
         self.g.postgresql.should_use_s3_to_create_replica = false
-        self.g.etcd.base_client_url = 'http://remote'
-        self.g.etcd.client_url
+        self.g.etcd.client._base_uri = 'http://remote'
         self.g.postgresql.data_directory_empty = true
         self.g.etcd.race = true
         self.g.initialize()

tests/test_ha.py

@@ -71,7 +71,7 @@ class TestHa(unittest.TestCase):
         requests.put = requests_put
         requests.delete = requests_delete
         self.p = MockPostgresql()
-        self.e = Etcd({'ttl': 30, 'host': 'remotehost', 'scope': 'test'})
+        self.e = Etcd({'ttl': 30, 'host': 'remotehost:2379', 'scope': 'test'})
         self.ha = Ha(self.p, self.e)
         self.ha.load_cluster_from_etcd()
         self.ha.cluster = Cluster(False, None, None, [])