gitops/patroni
1
0
Fork 0
mirror of https://github.com/outbackdingo/patroni.git synced 2026-08-25 14:53:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Introduce the failsafe key in DCS (#2485)

Browse Source 
Extracted from #2379
This commit is contained in:
Alexander Kukushkin
2022-12-13 11:35:06 +01:00
committed by GitHub
parent 6ad5fee99d
commit 92d3e1c167
16 changed files with 130 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
patroni/dcs/__init__.py
+29 -5
View File
@@ -444,7 +444,7 @@ class TimelineHistory(namedtuple('TimelineHistory', 'index,value,lines')):
return TimelineHistory(index, value, lines)
class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,failover,sync,history,slots')):
class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,failover,sync,history,slots,failsafe')):
"""Immutable object (namedtuple) which represents PostgreSQL cluster.
Consists of the following fields:
@@ -606,11 +606,11 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,f
@property
def timeline(self):
"""
>>> Cluster(0, 0, 0, 0, 0, 0, 0, 0, 0).timeline
>>> Cluster(0, 0, 0, 0, 0, 0, 0, 0, 0, None).timeline
0
>>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[]'), 0).timeline
>>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[]'), 0, None).timeline
1
>>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[["a"]]'), 0).timeline
>>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[["a"]]'), 0, None).timeline
0
"""
if self.history:
@@ -655,6 +655,7 @@ class AbstractDCS(object):
_STATUS = 'status' # JSON, contains "leader_lsn" and confirmed_flush_lsn of logical "slots" on the leader
_LEADER_OPTIME = _OPTIME + '/' + _LEADER # legacy
_SYNC = 'sync'
_FAILSAFE = 'failsafe'
def __init__(self, config):
"""
@@ -672,6 +673,7 @@ class AbstractDCS(object):
self._last_lsn = ''
self._last_seen = 0
self._last_status = {}
self._last_failsafe = {}
self.event = Event()
def client_path(self, path):
@@ -717,6 +719,10 @@ class AbstractDCS(object):
def sync_path(self):
return self.client_path(self._SYNC)
@property
def failsafe_path(self):
return self.client_path(self._FAILSAFE)
@abc.abstractmethod
def set_ttl(self, ttl):
"""Set the new ttl value for leader key"""
@@ -768,6 +774,8 @@ class AbstractDCS(object):
raise
self._last_seen = int(time.time())
self._last_status = {self._OPTIME: cluster.last_lsn, 'slots': cluster.slots}
self._last_failsafe = cluster.failsafe
with self._cluster_thread_lock:
self._cluster = cluster
@@ -810,6 +818,18 @@ class AbstractDCS(object):
self._last_lsn = value[self._OPTIME]
self._write_leader_optime(str(value[self._OPTIME]))
@abc.abstractmethod
def _write_failsafe(self, value):
"""Write current cluster topology to DCS that will be used by failsafe mechanism (if enabled).
:param value: failsafe topology serialized in JSON format
:returns: `!True` on success."""
def write_failsafe(self, value):
if not (isinstance(self._last_failsafe, dict) and deep_compare(self._last_failsafe, value))\
and self._write_failsafe(json.dumps(value, separators=(',', ':'))):
self._last_failsafe = value
@abc.abstractmethod
def _update_leader(self):
"""Update leader key (or session) ttl
@@ -821,7 +841,7 @@ class AbstractDCS(object):
If update fails due to DCS not being accessible or because it is not able to
process requests (hopefuly temporary), the ~DCSError exception should be raised."""
def update_leader(self, last_lsn, slots=None):
def update_leader(self, last_lsn, slots=None, failsafe=None):
"""Update leader key (or session) ttl and optime/leader
:param last_lsn: absolute WAL LSN in bytes
@@ -834,6 +854,10 @@ class AbstractDCS(object):
if slots:
status['slots'] = slots
self.write_status(status)
if ret and failsafe is not None:
self.write_failsafe(failsafe)
return ret
@abc.abstractmethod
patroni/dcs/consul.py
+13 -2
View File
@@ -389,9 +389,16 @@ class Consul(AbstractDCS):
sync = nodes.get(self._SYNC)
sync = SyncState.from_node(sync and sync['ModifyIndex'], sync and sync['Value'])
return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
# get failsafe topology
failsafe = nodes.get(self._FAILSAFE)
try:
failsafe = json.loads(failsafe['Value']) if failsafe else None
except Exception:
failsafe = None
return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
except NotFound:
return Cluster(None, None, None, None, [], None, None, None, None)
return Cluster(None, None, None, None, [], None, None, None, None, None)
except Exception:
logger.exception('get_cluster')
raise ConsulError('Consul is not responding properly')
@@ -554,6 +561,10 @@ class Consul(AbstractDCS):
def _write_status(self, value):
return self._client.kv.put(self.status_path, value)
@catch_consul_errors
def _write_failsafe(self, value):
return self._client.kv.put(self.failsafe_path, value)
@staticmethod
def _run_and_handle_exceptions(method, *args, **kwargs):
retry = kwargs.pop('retry', None)
patroni/dcs/etcd.py
+13 -2
View File
@@ -661,9 +661,16 @@ class Etcd(AbstractEtcd):
sync = nodes.get(self._SYNC)
sync = SyncState.from_node(sync and sync.modifiedIndex, sync and sync.value)
cluster = Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
# get failsafe topology
failsafe = nodes.get(self._FAILSAFE)
try:
failsafe = json.loads(failsafe.value) if failsafe else None
except Exception:
failsafe = None
cluster = Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
except etcd.EtcdKeyNotFound:
cluster = Cluster(None, None, None, None, [], None, None, None, None)
cluster = Cluster(None, None, None, None, [], None, None, None, None, None)
except Exception as e:
self._handle_exception(e, 'get_cluster', raise_ex=EtcdError('Etcd is not responding properly'))
self._has_failed = False
@@ -716,6 +723,10 @@ class Etcd(AbstractEtcd):
except etcd.EtcdKeyNotFound:
return self._do_attempt_to_acquire_leader()
@catch_etcd_errors
def _write_failsafe(self, value):
return self._client.set(self.failsafe_path, value)
@catch_return_false_exception
def _update_leader(self):
return self._run_and_handle_exceptions(self._do_update_leader, retry=None)
patroni/dcs/etcd3.py
+12 -1
View File
@@ -712,7 +712,14 @@ class Etcd3(AbstractEtcd):
sync = nodes.get(self._SYNC)
sync = SyncState.from_node(sync and sync['mod_revision'], sync and sync['value'])
cluster = Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
# get failsafe topology
failsafe = nodes.get(self._FAILSAFE)
try:
failsafe = json.loads(failsafe['value']) if failsafe else None
except Exception:
failsafe = None
cluster = Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
except UnsupportedEtcdVersion:
raise
except Exception as e:
@@ -801,6 +808,10 @@ class Etcd3(AbstractEtcd):
def _write_status(self, value):
return self._client.put(self.status_path, value)
@catch_etcd_errors
def _write_failsafe(self, value):
return self._client.put(self.failsafe_path, value)
@catch_return_false_exception
def _update_leader(self):
retry = self._retry.copy()
patroni/dcs/kubernetes.py
+16 -3
View File
@@ -832,6 +832,13 @@ class Kubernetes(AbstractDCS):
except Exception:
slots = None
# get failsafe topology
failsafe = annotations.get(self._FAILSAFE)
try:
failsafe = json.loads(failsafe) if failsafe else None
except Exception:
failsafe = None
# get leader
leader_record = {n: annotations.get(n) for n in (self._LEADER, 'acquireTime',
'ttl', 'renewTime', 'transitions') if n in annotations}
@@ -864,7 +871,7 @@ class Kubernetes(AbstractDCS):
metadata = sync and sync.metadata
sync = SyncState.from_node(metadata and metadata.resource_version, metadata and metadata.annotations)
return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
except Exception:
logger.exception('get_cluster')
raise KubernetesError('Kubernetes API is not responding properly')
@@ -999,6 +1006,9 @@ class Kubernetes(AbstractDCS):
def _write_status(self, value):
"""Unused"""
def _write_failsafe(self, value):
"""Unused"""
def _update_leader(self):
"""Unused"""
@@ -1050,7 +1060,7 @@ class Kubernetes(AbstractDCS):
return self._run_and_handle_exceptions(self._patch_or_create, self.leader_path, annotations,
kind_resource_version, ips=ips, retry=_retry)
def update_leader(self, last_lsn, slots=None):
def update_leader(self, last_lsn, slots=None, failsafe=None):
kind = self._kinds.get(self.leader_path)
kind_annotations = kind and kind.metadata.annotations or {}
@@ -1064,7 +1074,10 @@ class Kubernetes(AbstractDCS):
'transitions': leader_observed_record.get('transitions') or '0'}
if last_lsn:
annotations[self._OPTIME] = str(last_lsn)
annotations['slots'] = json.dumps(slots) if slots else None
annotations['slots'] = json.dumps(slots, separators=(',', ':')) if slots else None
if failsafe is not None:
annotations[self._FAILSAFE] = json.dumps(failsafe, separators=(',', ':')) if failsafe else None
resource_version = kind and kind.metadata.resource_version
return self._update_leader_with_retry(annotations, resource_version, self.__ips)
patroni/dcs/raft.py
+12 -2
View File
@@ -323,7 +323,7 @@ class Raft(AbstractDCS):
prefix = self.client_path('')
response = self._sync_obj.get(prefix, recursive=True)
if not response:
return Cluster(None, None, None, None, [], None, None, None, None)
return Cluster(None, None, None, None, [], None, None, None, None, None)
nodes = {os.path.relpath(key, prefix).replace('\\', '/'): value for key, value in response.items()}
# get initialize flag
@@ -376,7 +376,14 @@ class Raft(AbstractDCS):
sync = nodes.get(self._SYNC)
sync = SyncState.from_node(sync and sync['index'], sync and sync['value'])
return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
# get failsafe topology
failsafe = nodes.get(self._FAILSAFE)
try:
failsafe = json.loads(failsafe['value']) if failsafe else None
except Exception:
failsafe = None
return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
def _write_leader_optime(self, last_lsn):
return self._sync_obj.set(self.leader_optime_path, last_lsn, timeout=1)
@@ -384,6 +391,9 @@ class Raft(AbstractDCS):
def _write_status(self, value):
return self._sync_obj.set(self.status_path, value, timeout=1)
def _write_failsafe(self, value):
return self._sync_obj.set(self.failsafe_path, value, timeout=1)
def _update_leader(self):
ret = self._sync_obj.set(self.leader_path, self._name, ttl=self._ttl,
handle_raft_error=False, prevValue=self._name)
patroni/dcs/zookeeper.py
+13 -3
View File
@@ -276,7 +276,14 @@ class ZooKeeper(AbstractDCS):
failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
failover = failover and Failover.from_node(failover[1].version, failover[0])
return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots)
# get failsafe topology
failsafe = self.get_node(self.failsafe_path, watch=self.cluster_watcher) if self._FAILSAFE in nodes else None
try:
failsafe = json.loads(failsafe[0]) if failsafe else None
except Exception:
failsafe = None
return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe)
def _load_cluster(self):
cluster = self.cluster
@@ -297,8 +304,8 @@ class ZooKeeper(AbstractDCS):
try:
last_lsn, slots = self.get_status(cluster.leader)
self.event.clear()
cluster = Cluster(cluster.initialize, cluster.config, cluster.leader, last_lsn,
cluster.members, cluster.failover, cluster.sync, cluster.history, slots)
cluster = Cluster(cluster.initialize, cluster.config, cluster.leader, last_lsn, cluster.members,
cluster.failover, cluster.sync, cluster.history, slots, cluster.failsafe)
except Exception:
pass
return cluster
@@ -408,6 +415,9 @@ class ZooKeeper(AbstractDCS):
def _write_status(self, value):
return self._set_or_create(self.status_path, value)
def _write_failsafe(self, value):
return self._set_or_create(self.failsafe_path, value)
def _update_leader(self):
cluster = self.cluster
session = cluster and isinstance(cluster.leader, Leader) and cluster.leader.session
tests/test_consul.py
+3 -1
View File
@@ -34,6 +34,8 @@ def kv_get(self, key, **kwargs):
'ModifyIndex': 6429, 'Value': b'4496294792'},
{'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'sync', 'LockIndex': 0,
'ModifyIndex': 6429, 'Value': b'{"leader": "leader", "sync_standby": null}'},
{'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'failsafe', 'LockIndex': 0,
'ModifyIndex': 6429, 'Value': b'{'},
{'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'status', 'LockIndex': 0,
'ModifyIndex': 6429, 'Value': b'{"optime":4496294792, "slots":{"ls":12345}}'}])
if key == 'service/good/':
@@ -167,7 +169,7 @@ class TestConsul(unittest.TestCase):
self.c._session = 'fd4f44fe-2cac-bba5-a60b-304b51ff39b8'
with patch.object(consul.Consul.KV, 'delete', Mock(return_value=True)):
with patch.object(consul.Consul.KV, 'put', Mock(return_value=True)):
self.assertTrue(self.c.update_leader(12345))
self.assertTrue(self.c.update_leader(12345, failsafe={'foo': 'bar'}))
with patch.object(consul.Consul.KV, 'put', Mock(side_effect=ConsulException)):
self.assertFalse(self.c.update_leader(12345))
with patch('time.time', Mock(side_effect=[0, 0, 0, 0, 0, 100, 200, 300])):
tests/test_etcd.py
+2 -1
View File
@@ -67,6 +67,7 @@ def etcd_read(self, key, **kwargs):
"expiration": "2015-05-15T09:11:09.611860899Z", "ttl": 30,
"modifiedIndex": 20730, "createdIndex": 20730}],
"modifiedIndex": 1581, "createdIndex": 1581},
{"key": "/service/batman5/failsafe", "value": '{', "modifiedIndex": 1582, "createdIndex": 1582},
{"key": "/service/batman5/status", "value": '{"optime":2164261704,"slots":{"ls":12345}}',
"modifiedIndex": 1582, "createdIndex": 1582}], "modifiedIndex": 1581, "createdIndex": 1581}}
if key == '/service/legacy/':
@@ -286,7 +287,7 @@ class TestEtcd(unittest.TestCase):
self.etcd.write_leader_optime('0')
def test_update_leader(self):
self.assertTrue(self.etcd.update_leader(None))
self.assertTrue(self.etcd.update_leader(None, failsafe={'foo': 'bar'}))
with patch.object(etcd.Client, 'write',
Mock(side_effect=[etcd.EtcdConnectionFailed, etcd.EtcdClusterIdChanged, Exception])):
self.assertRaises(EtcdError, self.etcd.update_leader, None)
tests/test_etcd3.py
+3 -2
View File
@@ -36,7 +36,8 @@ def mock_urlopen(self, method, url, **kwargs):
"value": base64_encode('{}'), "lease": "123", "mod_revision": '1'},
{"key": base64_encode('/patroni/test/members/bar'),
"value": base64_encode('{"version":"1.6.5"}'), "lease": "123", "mod_revision": '1'},
{"key": base64_encode('/patroni/test/failover'), "value": base64_encode('{}'), "mod_revision": '1'}
{"key": base64_encode('/patroni/test/failover'), "value": base64_encode('{}'), "mod_revision": '1'},
{"key": base64_encode('/patroni/test/failsafe'), "value": base64_encode('{'), "mod_revision": '1'}
]
})
elif url.endswith('/watch'):
@@ -215,7 +216,7 @@ class TestEtcd3(BaseTestEtcd3):
def test__update_leader(self):
self.etcd3._lease = None
self.etcd3.update_leader('123')
self.etcd3.update_leader('123', failsafe={'foo': 'bar'})
self.etcd3._last_lease_refresh = 0
self.etcd3.update_leader('124')
with patch.object(PatroniEtcd3Client, 'lease_keepalive', Mock(return_value=True)),\
tests/test_ha.py
+1 -1
View File
@@ -38,7 +38,7 @@ def get_cluster(initialize, leader, members, failover, sync, cluster_config=None
history = TimelineHistory(1, '[[1,67197376,"no recovery target specified","' + t + '","foo"]]',
[(1, 67197376, 'no recovery target specified', t, 'foo')])
cluster_config = cluster_config or ClusterConfig(1, {'check_timeline': True}, 1)
return Cluster(initialize, cluster_config, leader, 10, members, failover, sync, history, None)
return Cluster(initialize, cluster_config, leader, 10, members, failover, sync, history, None, None)
def get_cluster_not_initialized_without_leader(cluster_config=None):
tests/test_kubernetes.py
+2 -2
View File
@@ -19,7 +19,7 @@ def mock_list_namespaced_config_map(*args, **kwargs):
'annotations': {'initialize': '123', 'config': '{}'}}
items = [k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))]
metadata.update({'name': 'test-leader',
'annotations': {'optime': '1234x', 'leader': 'p-0', 'ttl': '30s', 'slots': '{'}})
'annotations': {'optime': '1234x', 'leader': 'p-0', 'ttl': '30s', 'slots': '{', 'failsafe': '{'}})
items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
metadata.update({'name': 'test-failover', 'annotations': {'leader': 'p-0'}})
items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
@@ -300,7 +300,7 @@ class TestKubernetesEndpoints(BaseTestKubernetes):
@patch.object(k8s_client.CoreV1Api, 'patch_namespaced_endpoints', create=True)
def test_update_leader(self, mock_patch_namespaced_endpoints):
self.assertIsNotNone(self.k.update_leader('123'))
self.assertIsNotNone(self.k.update_leader('123', failsafe={'foo': 'bar'}))
args = mock_patch_namespaced_endpoints.call_args[0]
self.assertEqual(args[2].subsets[0].addresses[0].target_ref.resource_version, '10')
self.k._kinds._object_cache['test'].subsets[:] = []
tests/test_postgresql.py
+1 -1
View File
@@ -640,7 +640,7 @@ class TestPostgresql(BaseTestPostgresql):
def test_pick_sync_standby(self):
cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None,
SyncState(0, self.me.name, self.leadermem.name), None, None)
SyncState(0, self.me.name, self.leadermem.name), None, None, None)
mock_cursor = Mock()
mock_cursor.fetchone.return_value = ('remote_apply',)
tests/test_raft.py
+2 -1
View File
@@ -140,7 +140,8 @@ class TestRaft(unittest.TestCase):
raft.get_cluster()
self.assertTrue(raft._sync_obj.set(raft.status_path, '{"optime":1234567,"slots":{"ls":12345}}'))
raft.get_cluster()
self.assertTrue(raft.update_leader('1'))
self.assertTrue(raft.update_leader('1', failsafe={'foo': 'bat'}))
self.assertTrue(raft._sync_obj.set(raft.failsafe_path, '{"foo"}'))
self.assertTrue(raft._sync_obj.set(raft.status_path, '{'))
raft.get_cluster()
self.assertTrue(raft.delete_sync_state())
tests/test_slots.py
+4 -3
View File
@@ -31,14 +31,14 @@ class TestSlotsHandler(BaseTestPostgresql):
self.p.start()
config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}}}, 1)
self.cluster = Cluster(True, config, self.leader, 0,
[self.me, self.other, self.leadermem], None, None, None, {'ls': 12345})
[self.me, self.other, self.leadermem], None, None, None, {'ls': 12345}, None)
def test_sync_replication_slots(self):
config = ClusterConfig(1, {'slots': {'test_3': {'database': 'a', 'plugin': 'b'},
'A': 0, 'ls': 0, 'b': {'type': 'logical', 'plugin': '1'}},
'ignore_slots': [{'name': 'blabla'}]}, 1)
cluster = Cluster(True, config, self.leader, 0,
[self.me, self.other, self.leadermem], None, None, None, {'test_3': 10})
[self.me, self.other, self.leadermem], None, None, None, {'test_3': 10}, None)
with mock.patch('patroni.postgresql.Postgresql._query', Mock(side_effect=psycopg.OperationalError)):
self.s.sync_replication_slots(cluster, False)
self.p.set_role('standby_leader')
@@ -67,7 +67,8 @@ class TestSlotsHandler(BaseTestPostgresql):
def test_process_permanent_slots(self):
config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}},
'ignore_slots': [{'name': 'blabla'}]}, 1)
cluster = Cluster(True, config, self.leader, 0, [self.me, self.other, self.leadermem], None, None, None, None)
cluster = Cluster(True, config, self.leader, 0,
[self.me, self.other, self.leadermem], None, None, None, None, None)
self.s.sync_replication_slots(cluster, False)
with patch.object(Postgresql, '_query') as mock_query:
tests/test_zookeeper.py
+4 -2
View File
@@ -51,6 +51,8 @@ class MockKazooClient(Mock):
return (b'foo', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))
elif path.endswith('/status'):
return (b'{"optime":500,"slots":{"ls":1234567}}', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0))
elif path.endswith('/failsafe'):
return (b'{a}', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0))
return (b'', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))
@staticmethod
@@ -60,7 +62,7 @@ class MockKazooClient(Mock):
if path.startswith('/no_node'):
raise NoNodeError
elif path in ['/service/bla/', '/service/test/']:
return ['initialize', 'leader', 'members', 'optime', 'failover', 'sync']
return ['initialize', 'leader', 'members', 'optime', 'failover', 'sync', 'failsafe']
return ['foo', 'bar', 'buzz']
def create(self, path, value=b"", acl=None, ephemeral=False, sequence=False, makepath=False):
@@ -237,7 +239,7 @@ class TestZooKeeper(unittest.TestCase):
with patch.object(MockKazooClient, 'delete', Mock(side_effect=RetryFailedError)):
self.assertRaises(ZooKeeperError, self.zk.update_leader, 12345)
with patch.object(MockKazooClient, 'delete', Mock(side_effect=NoNodeError)):
self.assertTrue(self.zk.update_leader(12345))
self.assertTrue(self.zk.update_leader(12345, failsafe={'foo': 'bar'}))
with patch.object(MockKazooClient, 'create', Mock(side_effect=[RetryFailedError, Exception])):
self.assertRaises(ZooKeeperError, self.zk.update_leader, 12345)
self.assertFalse(self.zk.update_leader(12345))