From 66543f41a34e96cbe6466e5ca37e30cee9775f2d Mon Sep 17 00:00:00 2001
From: Alexander Kukushkin <cyberdemn@gmail.com>
Date: Fri, 4 Nov 2016 18:43:19 +0200
Subject: [PATCH] BUGFIX: Try all cluster members when doing pause/resume
 (#351)

Previously it was not possible to pause/resume unhealthy cluster.
---
 patroni/api.py    |  1 +
 patroni/ctl.py    | 23 ++++++++++++++++++-----
 tests/test_ctl.py |  4 ++++
 3 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/patroni/api.py b/patroni/api.py
index 17a84f15..92f931a7 100644
--- a/patroni/api.py
+++ b/patroni/api.py
@@ -141,6 +141,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 value = json.dumps(data, separators=(',', ':'))
                 if not self.server.patroni.dcs.set_config_value(value, cluster.config.index):
                     return self.send_error(409)
+            self.server.patroni.dcs.event.set()
             self._write_json_response(200, data)
 
     @check_auth
diff --git a/patroni/ctl.py b/patroni/ctl.py
index 068a39bb..07ed22b1 100644
--- a/patroni/ctl.py
+++ b/patroni/ctl.py
@@ -794,13 +794,26 @@ def toggle_pause(config, cluster_name, paused):
     if cluster.is_paused() == paused:
         raise PatroniCtlException('Cluster is {0} paused'.format(paused and 'already' or 'not'))
 
-    r = request_patroni(cluster.leader.member, 'patch', 'config', {'pause': paused or None}, auth_header(config))
+    members = []
+    if cluster.leader:
+        members.append(cluster.leader.member)
+    members.extend([m for m in cluster.members if m.api_url and (not members or members[0].name != m.name)])
 
-    if r.status_code == 200:
-        click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
+    for member in members:
+        try:
+            r = request_patroni(member, 'patch', 'config', {'pause': paused or None}, auth_header(config))
+        except Exception:
+            logging.warning('Member %s is not accessible', member.name)
+            continue
+
+        if r.status_code == 200:
+            click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
+        else:
+            click.echo('Failed: {0} cluster management status code={1}, ({2})'.format(
+                       paused and 'pause' or 'resume', r.status_code, r.text))
+        break
     else:
-        click.echo('Failed: {0} cluster management status code={1}, ({2})'.format(
-                   paused and 'pause' or 'resume', r.status_code, r.text))
+        raise PatroniCtlException('Can not find accessible cluster member')
 
 
 @ctl.command('pause', help='Disable auto failover')
diff --git a/tests/test_ctl.py b/tests/test_ctl.py
index 3cd01c7f..645df8f6 100644
--- a/tests/test_ctl.py
+++ b/tests/test_ctl.py
@@ -439,3 +439,7 @@ class TestCtl(unittest.TestCase):
                     patch('patroni.dcs.Cluster.is_paused', Mock(return_value=False)):
                 result = self.runner.invoke(ctl, ['resume', 'dummy'])
                 assert 'Cluster is not paused' in result.output
+
+            with patch('requests.patch', Mock(side_effect=Exception)):
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Can not find accessible cluster member' in result.output