diff --git a/patroni/dcs/__init__.py b/patroni/dcs/__init__.py index 0d414558..10242f0d 100644 --- a/patroni/dcs/__init__.py +++ b/patroni/dcs/__init__.py @@ -444,7 +444,7 @@ class TimelineHistory(namedtuple('TimelineHistory', 'index,value,lines')): return TimelineHistory(index, value, lines) -class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,failover,sync,history,slots')): +class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,failover,sync,history,slots,failsafe')): """Immutable object (namedtuple) which represents PostgreSQL cluster. Consists of the following fields: @@ -606,11 +606,11 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,f @property def timeline(self): """ - >>> Cluster(0, 0, 0, 0, 0, 0, 0, 0, 0).timeline + >>> Cluster(0, 0, 0, 0, 0, 0, 0, 0, 0, None).timeline 0 - >>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[]'), 0).timeline + >>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[]'), 0, None).timeline 1 - >>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[["a"]]'), 0).timeline + >>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[["a"]]'), 0, None).timeline 0 """ if self.history: @@ -655,6 +655,7 @@ class AbstractDCS(object): _STATUS = 'status' # JSON, contains "leader_lsn" and confirmed_flush_lsn of logical "slots" on the leader _LEADER_OPTIME = _OPTIME + '/' + _LEADER # legacy _SYNC = 'sync' + _FAILSAFE = 'failsafe' def __init__(self, config): """ @@ -672,6 +673,7 @@ class AbstractDCS(object): self._last_lsn = '' self._last_seen = 0 self._last_status = {} + self._last_failsafe = {} self.event = Event() def client_path(self, path): @@ -717,6 +719,10 @@ class AbstractDCS(object): def sync_path(self): return self.client_path(self._SYNC) + @property + def failsafe_path(self): + return self.client_path(self._FAILSAFE) + @abc.abstractmethod def set_ttl(self, ttl): """Set the new ttl value for leader key""" @@ -768,6 +774,8 @@ class AbstractDCS(object): raise self._last_seen = int(time.time()) + self._last_status = {self._OPTIME: cluster.last_lsn, 'slots': cluster.slots} + self._last_failsafe = cluster.failsafe with self._cluster_thread_lock: self._cluster = cluster @@ -810,6 +818,18 @@ class AbstractDCS(object): self._last_lsn = value[self._OPTIME] self._write_leader_optime(str(value[self._OPTIME])) + @abc.abstractmethod + def _write_failsafe(self, value): + """Write current cluster topology to DCS that will be used by failsafe mechanism (if enabled). + + :param value: failsafe topology serialized in JSON format + :returns: `!True` on success.""" + + def write_failsafe(self, value): + if not (isinstance(self._last_failsafe, dict) and deep_compare(self._last_failsafe, value))\ + and self._write_failsafe(json.dumps(value, separators=(',', ':'))): + self._last_failsafe = value + @abc.abstractmethod def _update_leader(self): """Update leader key (or session) ttl @@ -821,7 +841,7 @@ class AbstractDCS(object): If update fails due to DCS not being accessible or because it is not able to process requests (hopefuly temporary), the ~DCSError exception should be raised.""" - def update_leader(self, last_lsn, slots=None): + def update_leader(self, last_lsn, slots=None, failsafe=None): """Update leader key (or session) ttl and optime/leader :param last_lsn: absolute WAL LSN in bytes @@ -834,6 +854,10 @@ class AbstractDCS(object): if slots: status['slots'] = slots self.write_status(status) + + if ret and failsafe is not None: + self.write_failsafe(failsafe) + return ret @abc.abstractmethod diff --git a/patroni/dcs/consul.py b/patroni/dcs/consul.py index 1d506d4e..eb1a180b 100644 --- a/patroni/dcs/consul.py +++ b/patroni/dcs/consul.py @@ -389,9 +389,16 @@ class Consul(AbstractDCS): sync = nodes.get(self._SYNC) sync = SyncState.from_node(sync and sync['ModifyIndex'], sync and sync['Value']) - return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots) + # get failsafe topology + failsafe = nodes.get(self._FAILSAFE) + try: + failsafe = json.loads(failsafe['Value']) if failsafe else None + except Exception: + failsafe = None + + return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe) except NotFound: - return Cluster(None, None, None, None, [], None, None, None, None) + return Cluster(None, None, None, None, [], None, None, None, None, None) except Exception: logger.exception('get_cluster') raise ConsulError('Consul is not responding properly') @@ -554,6 +561,10 @@ class Consul(AbstractDCS): def _write_status(self, value): return self._client.kv.put(self.status_path, value) + @catch_consul_errors + def _write_failsafe(self, value): + return self._client.kv.put(self.failsafe_path, value) + @staticmethod def _run_and_handle_exceptions(method, *args, **kwargs): retry = kwargs.pop('retry', None) diff --git a/patroni/dcs/etcd.py b/patroni/dcs/etcd.py index 2921b8e1..3a8cf0c1 100644 --- a/patroni/dcs/etcd.py +++ b/patroni/dcs/etcd.py @@ -661,9 +661,16 @@ class Etcd(AbstractEtcd): sync = nodes.get(self._SYNC) sync = SyncState.from_node(sync and sync.modifiedIndex, sync and sync.value) - cluster = Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots) + # get failsafe topology + failsafe = nodes.get(self._FAILSAFE) + try: + failsafe = json.loads(failsafe.value) if failsafe else None + except Exception: + failsafe = None + + cluster = Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe) except etcd.EtcdKeyNotFound: - cluster = Cluster(None, None, None, None, [], None, None, None, None) + cluster = Cluster(None, None, None, None, [], None, None, None, None, None) except Exception as e: self._handle_exception(e, 'get_cluster', raise_ex=EtcdError('Etcd is not responding properly')) self._has_failed = False @@ -716,6 +723,10 @@ class Etcd(AbstractEtcd): except etcd.EtcdKeyNotFound: return self._do_attempt_to_acquire_leader() + @catch_etcd_errors + def _write_failsafe(self, value): + return self._client.set(self.failsafe_path, value) + @catch_return_false_exception def _update_leader(self): return self._run_and_handle_exceptions(self._do_update_leader, retry=None) diff --git a/patroni/dcs/etcd3.py b/patroni/dcs/etcd3.py index 3403f323..0c2d144b 100644 --- a/patroni/dcs/etcd3.py +++ b/patroni/dcs/etcd3.py @@ -712,7 +712,14 @@ class Etcd3(AbstractEtcd): sync = nodes.get(self._SYNC) sync = SyncState.from_node(sync and sync['mod_revision'], sync and sync['value']) - cluster = Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots) + # get failsafe topology + failsafe = nodes.get(self._FAILSAFE) + try: + failsafe = json.loads(failsafe['value']) if failsafe else None + except Exception: + failsafe = None + + cluster = Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe) except UnsupportedEtcdVersion: raise except Exception as e: @@ -801,6 +808,10 @@ class Etcd3(AbstractEtcd): def _write_status(self, value): return self._client.put(self.status_path, value) + @catch_etcd_errors + def _write_failsafe(self, value): + return self._client.put(self.failsafe_path, value) + @catch_return_false_exception def _update_leader(self): retry = self._retry.copy() diff --git a/patroni/dcs/kubernetes.py b/patroni/dcs/kubernetes.py index a0fe3dfe..67541a34 100644 --- a/patroni/dcs/kubernetes.py +++ b/patroni/dcs/kubernetes.py @@ -832,6 +832,13 @@ class Kubernetes(AbstractDCS): except Exception: slots = None + # get failsafe topology + failsafe = annotations.get(self._FAILSAFE) + try: + failsafe = json.loads(failsafe) if failsafe else None + except Exception: + failsafe = None + # get leader leader_record = {n: annotations.get(n) for n in (self._LEADER, 'acquireTime', 'ttl', 'renewTime', 'transitions') if n in annotations} @@ -864,7 +871,7 @@ class Kubernetes(AbstractDCS): metadata = sync and sync.metadata sync = SyncState.from_node(metadata and metadata.resource_version, metadata and metadata.annotations) - return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots) + return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe) except Exception: logger.exception('get_cluster') raise KubernetesError('Kubernetes API is not responding properly') @@ -999,6 +1006,9 @@ class Kubernetes(AbstractDCS): def _write_status(self, value): """Unused""" + def _write_failsafe(self, value): + """Unused""" + def _update_leader(self): """Unused""" @@ -1050,7 +1060,7 @@ class Kubernetes(AbstractDCS): return self._run_and_handle_exceptions(self._patch_or_create, self.leader_path, annotations, kind_resource_version, ips=ips, retry=_retry) - def update_leader(self, last_lsn, slots=None): + def update_leader(self, last_lsn, slots=None, failsafe=None): kind = self._kinds.get(self.leader_path) kind_annotations = kind and kind.metadata.annotations or {} @@ -1064,7 +1074,10 @@ class Kubernetes(AbstractDCS): 'transitions': leader_observed_record.get('transitions') or '0'} if last_lsn: annotations[self._OPTIME] = str(last_lsn) - annotations['slots'] = json.dumps(slots) if slots else None + annotations['slots'] = json.dumps(slots, separators=(',', ':')) if slots else None + + if failsafe is not None: + annotations[self._FAILSAFE] = json.dumps(failsafe, separators=(',', ':')) if failsafe else None resource_version = kind and kind.metadata.resource_version return self._update_leader_with_retry(annotations, resource_version, self.__ips) diff --git a/patroni/dcs/raft.py b/patroni/dcs/raft.py index 1318c188..8c563906 100644 --- a/patroni/dcs/raft.py +++ b/patroni/dcs/raft.py @@ -323,7 +323,7 @@ class Raft(AbstractDCS): prefix = self.client_path('') response = self._sync_obj.get(prefix, recursive=True) if not response: - return Cluster(None, None, None, None, [], None, None, None, None) + return Cluster(None, None, None, None, [], None, None, None, None, None) nodes = {os.path.relpath(key, prefix).replace('\\', '/'): value for key, value in response.items()} # get initialize flag @@ -376,7 +376,14 @@ class Raft(AbstractDCS): sync = nodes.get(self._SYNC) sync = SyncState.from_node(sync and sync['index'], sync and sync['value']) - return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots) + # get failsafe topology + failsafe = nodes.get(self._FAILSAFE) + try: + failsafe = json.loads(failsafe['value']) if failsafe else None + except Exception: + failsafe = None + + return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe) def _write_leader_optime(self, last_lsn): return self._sync_obj.set(self.leader_optime_path, last_lsn, timeout=1) @@ -384,6 +391,9 @@ class Raft(AbstractDCS): def _write_status(self, value): return self._sync_obj.set(self.status_path, value, timeout=1) + def _write_failsafe(self, value): + return self._sync_obj.set(self.failsafe_path, value, timeout=1) + def _update_leader(self): ret = self._sync_obj.set(self.leader_path, self._name, ttl=self._ttl, handle_raft_error=False, prevValue=self._name) diff --git a/patroni/dcs/zookeeper.py b/patroni/dcs/zookeeper.py index 9101244f..c4f686a6 100644 --- a/patroni/dcs/zookeeper.py +++ b/patroni/dcs/zookeeper.py @@ -276,7 +276,14 @@ class ZooKeeper(AbstractDCS): failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None failover = failover and Failover.from_node(failover[1].version, failover[0]) - return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots) + # get failsafe topology + failsafe = self.get_node(self.failsafe_path, watch=self.cluster_watcher) if self._FAILSAFE in nodes else None + try: + failsafe = json.loads(failsafe[0]) if failsafe else None + except Exception: + failsafe = None + + return Cluster(initialize, config, leader, last_lsn, members, failover, sync, history, slots, failsafe) def _load_cluster(self): cluster = self.cluster @@ -297,8 +304,8 @@ class ZooKeeper(AbstractDCS): try: last_lsn, slots = self.get_status(cluster.leader) self.event.clear() - cluster = Cluster(cluster.initialize, cluster.config, cluster.leader, last_lsn, - cluster.members, cluster.failover, cluster.sync, cluster.history, slots) + cluster = Cluster(cluster.initialize, cluster.config, cluster.leader, last_lsn, cluster.members, + cluster.failover, cluster.sync, cluster.history, slots, cluster.failsafe) except Exception: pass return cluster @@ -408,6 +415,9 @@ class ZooKeeper(AbstractDCS): def _write_status(self, value): return self._set_or_create(self.status_path, value) + def _write_failsafe(self, value): + return self._set_or_create(self.failsafe_path, value) + def _update_leader(self): cluster = self.cluster session = cluster and isinstance(cluster.leader, Leader) and cluster.leader.session diff --git a/tests/test_consul.py b/tests/test_consul.py index 03c90273..caf99bb0 100644 --- a/tests/test_consul.py +++ b/tests/test_consul.py @@ -34,6 +34,8 @@ def kv_get(self, key, **kwargs): 'ModifyIndex': 6429, 'Value': b'4496294792'}, {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'sync', 'LockIndex': 0, 'ModifyIndex': 6429, 'Value': b'{"leader": "leader", "sync_standby": null}'}, + {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'failsafe', 'LockIndex': 0, + 'ModifyIndex': 6429, 'Value': b'{'}, {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'status', 'LockIndex': 0, 'ModifyIndex': 6429, 'Value': b'{"optime":4496294792, "slots":{"ls":12345}}'}]) if key == 'service/good/': @@ -167,7 +169,7 @@ class TestConsul(unittest.TestCase): self.c._session = 'fd4f44fe-2cac-bba5-a60b-304b51ff39b8' with patch.object(consul.Consul.KV, 'delete', Mock(return_value=True)): with patch.object(consul.Consul.KV, 'put', Mock(return_value=True)): - self.assertTrue(self.c.update_leader(12345)) + self.assertTrue(self.c.update_leader(12345, failsafe={'foo': 'bar'})) with patch.object(consul.Consul.KV, 'put', Mock(side_effect=ConsulException)): self.assertFalse(self.c.update_leader(12345)) with patch('time.time', Mock(side_effect=[0, 0, 0, 0, 0, 100, 200, 300])): diff --git a/tests/test_etcd.py b/tests/test_etcd.py index d22d9760..61980474 100644 --- a/tests/test_etcd.py +++ b/tests/test_etcd.py @@ -67,6 +67,7 @@ def etcd_read(self, key, **kwargs): "expiration": "2015-05-15T09:11:09.611860899Z", "ttl": 30, "modifiedIndex": 20730, "createdIndex": 20730}], "modifiedIndex": 1581, "createdIndex": 1581}, + {"key": "/service/batman5/failsafe", "value": '{', "modifiedIndex": 1582, "createdIndex": 1582}, {"key": "/service/batman5/status", "value": '{"optime":2164261704,"slots":{"ls":12345}}', "modifiedIndex": 1582, "createdIndex": 1582}], "modifiedIndex": 1581, "createdIndex": 1581}} if key == '/service/legacy/': @@ -286,7 +287,7 @@ class TestEtcd(unittest.TestCase): self.etcd.write_leader_optime('0') def test_update_leader(self): - self.assertTrue(self.etcd.update_leader(None)) + self.assertTrue(self.etcd.update_leader(None, failsafe={'foo': 'bar'})) with patch.object(etcd.Client, 'write', Mock(side_effect=[etcd.EtcdConnectionFailed, etcd.EtcdClusterIdChanged, Exception])): self.assertRaises(EtcdError, self.etcd.update_leader, None) diff --git a/tests/test_etcd3.py b/tests/test_etcd3.py index e09c59bc..a4ba5841 100644 --- a/tests/test_etcd3.py +++ b/tests/test_etcd3.py @@ -36,7 +36,8 @@ def mock_urlopen(self, method, url, **kwargs): "value": base64_encode('{}'), "lease": "123", "mod_revision": '1'}, {"key": base64_encode('/patroni/test/members/bar'), "value": base64_encode('{"version":"1.6.5"}'), "lease": "123", "mod_revision": '1'}, - {"key": base64_encode('/patroni/test/failover'), "value": base64_encode('{}'), "mod_revision": '1'} + {"key": base64_encode('/patroni/test/failover'), "value": base64_encode('{}'), "mod_revision": '1'}, + {"key": base64_encode('/patroni/test/failsafe'), "value": base64_encode('{'), "mod_revision": '1'} ] }) elif url.endswith('/watch'): @@ -215,7 +216,7 @@ class TestEtcd3(BaseTestEtcd3): def test__update_leader(self): self.etcd3._lease = None - self.etcd3.update_leader('123') + self.etcd3.update_leader('123', failsafe={'foo': 'bar'}) self.etcd3._last_lease_refresh = 0 self.etcd3.update_leader('124') with patch.object(PatroniEtcd3Client, 'lease_keepalive', Mock(return_value=True)),\ diff --git a/tests/test_ha.py b/tests/test_ha.py index 7a7fb5f0..4ec0d7f7 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -38,7 +38,7 @@ def get_cluster(initialize, leader, members, failover, sync, cluster_config=None history = TimelineHistory(1, '[[1,67197376,"no recovery target specified","' + t + '","foo"]]', [(1, 67197376, 'no recovery target specified', t, 'foo')]) cluster_config = cluster_config or ClusterConfig(1, {'check_timeline': True}, 1) - return Cluster(initialize, cluster_config, leader, 10, members, failover, sync, history, None) + return Cluster(initialize, cluster_config, leader, 10, members, failover, sync, history, None, None) def get_cluster_not_initialized_without_leader(cluster_config=None): diff --git a/tests/test_kubernetes.py b/tests/test_kubernetes.py index 9ff376a9..2dd1fd70 100644 --- a/tests/test_kubernetes.py +++ b/tests/test_kubernetes.py @@ -19,7 +19,7 @@ def mock_list_namespaced_config_map(*args, **kwargs): 'annotations': {'initialize': '123', 'config': '{}'}} items = [k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))] metadata.update({'name': 'test-leader', - 'annotations': {'optime': '1234x', 'leader': 'p-0', 'ttl': '30s', 'slots': '{'}}) + 'annotations': {'optime': '1234x', 'leader': 'p-0', 'ttl': '30s', 'slots': '{', 'failsafe': '{'}}) items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))) metadata.update({'name': 'test-failover', 'annotations': {'leader': 'p-0'}}) items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))) @@ -300,7 +300,7 @@ class TestKubernetesEndpoints(BaseTestKubernetes): @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_endpoints', create=True) def test_update_leader(self, mock_patch_namespaced_endpoints): - self.assertIsNotNone(self.k.update_leader('123')) + self.assertIsNotNone(self.k.update_leader('123', failsafe={'foo': 'bar'})) args = mock_patch_namespaced_endpoints.call_args[0] self.assertEqual(args[2].subsets[0].addresses[0].target_ref.resource_version, '10') self.k._kinds._object_cache['test'].subsets[:] = [] diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py index 9ee2d9a2..df24ba55 100644 --- a/tests/test_postgresql.py +++ b/tests/test_postgresql.py @@ -640,7 +640,7 @@ class TestPostgresql(BaseTestPostgresql): def test_pick_sync_standby(self): cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None, - SyncState(0, self.me.name, self.leadermem.name), None, None) + SyncState(0, self.me.name, self.leadermem.name), None, None, None) mock_cursor = Mock() mock_cursor.fetchone.return_value = ('remote_apply',) diff --git a/tests/test_raft.py b/tests/test_raft.py index 9797ec0c..2c8ce84e 100644 --- a/tests/test_raft.py +++ b/tests/test_raft.py @@ -140,7 +140,8 @@ class TestRaft(unittest.TestCase): raft.get_cluster() self.assertTrue(raft._sync_obj.set(raft.status_path, '{"optime":1234567,"slots":{"ls":12345}}')) raft.get_cluster() - self.assertTrue(raft.update_leader('1')) + self.assertTrue(raft.update_leader('1', failsafe={'foo': 'bat'})) + self.assertTrue(raft._sync_obj.set(raft.failsafe_path, '{"foo"}')) self.assertTrue(raft._sync_obj.set(raft.status_path, '{')) raft.get_cluster() self.assertTrue(raft.delete_sync_state()) diff --git a/tests/test_slots.py b/tests/test_slots.py index f96bc6d5..4b19fc4a 100644 --- a/tests/test_slots.py +++ b/tests/test_slots.py @@ -31,14 +31,14 @@ class TestSlotsHandler(BaseTestPostgresql): self.p.start() config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}}}, 1) self.cluster = Cluster(True, config, self.leader, 0, - [self.me, self.other, self.leadermem], None, None, None, {'ls': 12345}) + [self.me, self.other, self.leadermem], None, None, None, {'ls': 12345}, None) def test_sync_replication_slots(self): config = ClusterConfig(1, {'slots': {'test_3': {'database': 'a', 'plugin': 'b'}, 'A': 0, 'ls': 0, 'b': {'type': 'logical', 'plugin': '1'}}, 'ignore_slots': [{'name': 'blabla'}]}, 1) cluster = Cluster(True, config, self.leader, 0, - [self.me, self.other, self.leadermem], None, None, None, {'test_3': 10}) + [self.me, self.other, self.leadermem], None, None, None, {'test_3': 10}, None) with mock.patch('patroni.postgresql.Postgresql._query', Mock(side_effect=psycopg.OperationalError)): self.s.sync_replication_slots(cluster, False) self.p.set_role('standby_leader') @@ -67,7 +67,8 @@ class TestSlotsHandler(BaseTestPostgresql): def test_process_permanent_slots(self): config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}}, 'ignore_slots': [{'name': 'blabla'}]}, 1) - cluster = Cluster(True, config, self.leader, 0, [self.me, self.other, self.leadermem], None, None, None, None) + cluster = Cluster(True, config, self.leader, 0, + [self.me, self.other, self.leadermem], None, None, None, None, None) self.s.sync_replication_slots(cluster, False) with patch.object(Postgresql, '_query') as mock_query: diff --git a/tests/test_zookeeper.py b/tests/test_zookeeper.py index effcf6de..e1e2ac66 100644 --- a/tests/test_zookeeper.py +++ b/tests/test_zookeeper.py @@ -51,6 +51,8 @@ class MockKazooClient(Mock): return (b'foo', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)) elif path.endswith('/status'): return (b'{"optime":500,"slots":{"ls":1234567}}', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0)) + elif path.endswith('/failsafe'): + return (b'{a}', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0)) return (b'', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)) @staticmethod @@ -60,7 +62,7 @@ class MockKazooClient(Mock): if path.startswith('/no_node'): raise NoNodeError elif path in ['/service/bla/', '/service/test/']: - return ['initialize', 'leader', 'members', 'optime', 'failover', 'sync'] + return ['initialize', 'leader', 'members', 'optime', 'failover', 'sync', 'failsafe'] return ['foo', 'bar', 'buzz'] def create(self, path, value=b"", acl=None, ephemeral=False, sequence=False, makepath=False): @@ -237,7 +239,7 @@ class TestZooKeeper(unittest.TestCase): with patch.object(MockKazooClient, 'delete', Mock(side_effect=RetryFailedError)): self.assertRaises(ZooKeeperError, self.zk.update_leader, 12345) with patch.object(MockKazooClient, 'delete', Mock(side_effect=NoNodeError)): - self.assertTrue(self.zk.update_leader(12345)) + self.assertTrue(self.zk.update_leader(12345, failsafe={'foo': 'bar'})) with patch.object(MockKazooClient, 'create', Mock(side_effect=[RetryFailedError, Exception])): self.assertRaises(ZooKeeperError, self.zk.update_leader, 12345) self.assertFalse(self.zk.update_leader(12345))