Merge pull request #383 from zalando/bugfix/etcd-machines

Call self._load_machines_cache() method on timeout is causing switch to a new server every 5 minutes

patroni/dcs/etcd.py

@@ -194,8 +194,13 @@ class Client(etcd.Client):
             raise etcd.EtcdException('HTTP method {0} not supported'.format(method))
 
         # Update machines_cache if previous attempt of update has failed
-        if self._update_machines_cache or time.time() - self._machines_cache_updated > self._machines_cache_ttl:
+        if self._update_machines_cache:
             self._load_machines_cache()
+        elif time.time() - self._machines_cache_updated > self._machines_cache_ttl:
+            self._machines_cache = self.machines
+            if self._base_uri_unresolved in self._machines_cache:
+                self._machines_cache.remove(self._base_uri_unresolved)
+            self._machines_cache_updated = time.time()
 
         kwargs.update(self._build_request_parameters())
 

tests/test_etcd.py

@@ -192,6 +192,9 @@ class TestClient(unittest.TestCase):
         self.client._base_uri = 'http://localhost:4001'
         self.client._machines_cache = ['http://localhost:2379']
         self.client.api_execute('/', 'POST', timeout=0)
+        mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
+        self.client._machines_cache_updated = 0
+        self.client.api_execute('/', 'POST', timeout=0)
         self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
         self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
         self.client._update_machines_cache = True