gitops/patroni
1
0
Fork 0
mirror of https://github.com/outbackdingo/patroni.git synced 2026-09-02 01:29:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Raise DCSError when communication with DCS fails (#2484)

Browse Source 
Previously such an exception was raised only from the `get_cluster()` method, and now we will to do the same from the `update_leader()` and `attempt_to_acquire_leader()` methods.

These methods influence Postgres promotion and demotion and we want to make a difference between different types of failures. Specifically, if calls have failed because DCS isn't accessible or due to a timeout.

This commit is extracted from the #2379
This commit is contained in:
Alexander Kukushkin
2022-12-13 11:06:55 +01:00
committed by GitHub
parent 78d3f2cac2
commit 6ad5fee99d
16 changed files with 364 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_consul.py
+27 -10
View File
@@ -4,7 +4,7 @@ import unittest
from consul import ConsulException, NotFound
from mock import Mock, PropertyMock, patch
from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulInternalError, \
ConsulError, ConsulClient, HTTPClient, InvalidSessionTTL, InvalidSession
ConsulError, ConsulClient, HTTPClient, InvalidSessionTTL, InvalidSession, RetryFailedError
from . import SleepException
@@ -122,9 +122,6 @@ class TestConsul(unittest.TestCase):
self.assertIsInstance(self.c.get_cluster(), Cluster)
self.c._base_path = '/service/legacy'
self.assertIsInstance(self.c.get_cluster(), Cluster)
self.c._base_path = '/service/good'
self.c._session = 'fd4f44fe-2cac-bba5-a60b-304b51ff39b8'
self.assertIsInstance(self.c.get_cluster(), Cluster)
@patch.object(consul.Consul.KV, 'delete', Mock(side_effect=[ConsulException, True, True, True]))
@patch.object(consul.Consul.KV, 'put', Mock(side_effect=[True, ConsulException, InvalidSession]))
@@ -140,11 +137,15 @@ class TestConsul(unittest.TestCase):
self.c.refresh_session = Mock(side_effect=ConsulError('foo'))
self.assertFalse(self.c.touch_member({'balbla': 'blabla'}))
@patch.object(consul.Consul.KV, 'put', Mock(side_effect=InvalidSession))
@patch.object(consul.Consul.KV, 'put', Mock(side_effect=[InvalidSession, False, InvalidSession]))
def test_take_leader(self):
self.c.set_ttl(20)
self.c.refresh_session = Mock()
self.c.take_leader()
self.c._do_refresh_session = Mock()
self.assertFalse(self.c.take_leader())
with patch('time.time', Mock(side_effect=[0, 100])):
self.assertRaises(ConsulError, self.c.take_leader)
with patch('time.time', Mock(side_effect=[0, 0, 0, 0, 0, 0, 100])):
self.assertRaises(ConsulError, self.c.take_leader)
@patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
def test_set_failover_value(self):
@@ -160,10 +161,26 @@ class TestConsul(unittest.TestCase):
self.c.get_cluster()
self.c.write_leader_optime('1')
@patch.object(consul.Consul.Session, 'renew', Mock())
@patch.object(consul.Consul.Session, 'renew')
@patch.object(consul.Consul.KV, 'put', Mock(side_effect=ConsulException))
def test_update_leader(self):
self.c.update_leader(12345)
def test_update_leader(self, mock_renew):
self.c._session = 'fd4f44fe-2cac-bba5-a60b-304b51ff39b8'
with patch.object(consul.Consul.KV, 'delete', Mock(return_value=True)):
with patch.object(consul.Consul.KV, 'put', Mock(return_value=True)):
self.assertTrue(self.c.update_leader(12345))
with patch.object(consul.Consul.KV, 'put', Mock(side_effect=ConsulException)):
self.assertFalse(self.c.update_leader(12345))
with patch('time.time', Mock(side_effect=[0, 0, 0, 0, 0, 100, 200, 300])):
self.assertRaises(ConsulError, self.c.update_leader, 12345)
with patch('time.time', Mock(side_effect=[0, 100, 200, 300])):
self.assertRaises(ConsulError, self.c.update_leader, 12345)
with patch.object(consul.Consul.KV, 'delete', Mock(side_effect=ConsulException)):
self.assertFalse(self.c.update_leader(12347))
mock_renew.side_effect = RetryFailedError('')
self.c._last_session_refresh = 0
self.assertRaises(ConsulError, self.c.update_leader, 12346)
mock_renew.side_effect = ConsulException
self.assertFalse(self.c.update_leader(12347))
@patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
def test_delete_leader(self):
tests/test_etcd.py
+10
View File
@@ -276,6 +276,9 @@ class TestEtcd(unittest.TestCase):
self.assertFalse(self.etcd.attempt_to_acquire_leader())
self.etcd._base_path = '/service/failed'
self.assertFalse(self.etcd.attempt_to_acquire_leader())
with patch.object(EtcdClient, 'write', Mock(side_effect=[etcd.EtcdConnectionFailed, Exception])):
self.assertRaises(EtcdError, self.etcd.attempt_to_acquire_leader)
self.assertRaises(EtcdError, self.etcd.attempt_to_acquire_leader)
@patch.object(Cluster, 'min_version', PropertyMock(return_value=(2, 0)))
def test_write_leader_optime(self):
@@ -284,6 +287,13 @@ class TestEtcd(unittest.TestCase):
def test_update_leader(self):
self.assertTrue(self.etcd.update_leader(None))
with patch.object(etcd.Client, 'write',
Mock(side_effect=[etcd.EtcdConnectionFailed, etcd.EtcdClusterIdChanged, Exception])):
self.assertRaises(EtcdError, self.etcd.update_leader, None)
self.assertFalse(self.etcd.update_leader(None))
self.assertRaises(EtcdError, self.etcd.update_leader, None)
with patch.object(etcd.Client, 'write', Mock(side_effect=etcd.EtcdKeyNotFound)):
self.assertFalse(self.etcd.update_leader(None))
def test_initialize(self):
self.assertFalse(self.etcd.initialize())
tests/test_etcd3.py
+16 -1
View File
@@ -216,11 +216,26 @@ class TestEtcd3(BaseTestEtcd3):
def test__update_leader(self):
self.etcd3._lease = None
self.etcd3.update_leader('123')
self.etcd3._last_lease_refresh = 0
self.etcd3.update_leader('124')
with patch.object(PatroniEtcd3Client, 'lease_keepalive', Mock(return_value=True)),\
patch('time.time', Mock(side_effect=[0, 100, 200, 300])):
self.assertRaises(Etcd3Error, self.etcd3.update_leader, '126')
self.etcd3._last_lease_refresh = 0
with patch.object(PatroniEtcd3Client, 'lease_keepalive', Mock(side_effect=Unknown)):
self.assertFalse(self.etcd3.update_leader('125'))
def test_take_leader(self):
self.assertFalse(self.etcd3.take_leader())
def test_attempt_to_acquire_leader(self):
self.etcd3._lease = None
self.assertFalse(self.etcd3.attempt_to_acquire_leader())
with patch('time.time', Mock(side_effect=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 200])):
self.assertRaises(Etcd3Error, self.etcd3.attempt_to_acquire_leader)
with patch('time.time', Mock(side_effect=[0, 100, 200, 300, 400])):
self.assertRaises(Etcd3Error, self.etcd3.attempt_to_acquire_leader)
with patch.object(PatroniEtcd3Client, 'put', Mock(return_value=False)):
self.assertFalse(self.etcd3.attempt_to_acquire_leader())
def test_set_ttl(self):
self.etcd3.set_ttl(20)
tests/test_exhibitor.py
+1
View File
@@ -30,5 +30,6 @@ class TestExhibitor(unittest.TestCase):
'name': 'foo', 'ttl': 30, 'retry_timeout': 10})
@patch.object(ExhibitorEnsembleProvider, 'poll', Mock(return_value=True))
@patch.object(MockKazooClient, 'get_children', Mock(side_effect=Exception))
def test_get_cluster(self):
self.assertRaises(ZooKeeperError, self.e.get_cluster)
tests/test_ha.py
+10 -2
View File
@@ -206,7 +206,8 @@ class TestHa(PostgresInit):
def test_update_lock(self):
self.p.last_operation = Mock(side_effect=PostgresConnectionException(''))
self.ha.dcs.update_leader = Mock(side_effect=Exception)
self.ha.dcs.update_leader = Mock(side_effect=[DCSError(''), Exception])
self.assertRaises(DCSError, self.ha.update_lock)
self.assertFalse(self.ha.update_lock(True))
@patch.object(Postgresql, 'received_timeline', Mock(return_value=None))
@@ -458,7 +459,9 @@ class TestHa(PostgresInit):
def test_no_etcd_connection_master_demote(self):
self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
self.assertEqual(self.ha.run_cycle(), 'demoted self because DCS is not accessible and i was a leader')
self.assertEqual(self.ha.run_cycle(), 'demoting self because DCS is not accessible and I was a leader')
self.ha._async_executor.schedule('dummy')
self.assertEqual(self.ha.run_cycle(), 'demoted self because DCS is not accessible and I was a leader')
@patch('time.sleep', Mock())
def test_bootstrap_from_another_member(self):
@@ -1285,3 +1288,8 @@ class TestHa(PostgresInit):
self.ha.fetch_node_status = Mock(return_value=_MemberStatus(self.ha.cluster.members[0],
True, True, 0, 2, None, {}, False))
self.assertFalse(self.ha.is_failover_possible(self.ha.cluster.members))
def test_acquire_lock(self):
self.ha.dcs.attempt_to_acquire_leader = Mock(side_effect=[DCSError('foo'), Exception])
self.assertRaises(DCSError, self.ha.acquire_lock)
self.assertFalse(self.ha.acquire_lock())
tests/test_kubernetes.py
+10 -1
View File
@@ -238,6 +238,13 @@ class TestKubernetesConfigMaps(BaseTestKubernetes):
with patch.object(Kubernetes, '_wait_caches', Mock(side_effect=Exception)):
self.assertRaises(KubernetesError, self.k.get_cluster)
def test_attempt_to_acquire_leader(self):
with patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', create=True) as mock_patch:
mock_patch.side_effect = K8sException
self.assertRaises(KubernetesError, self.k.attempt_to_acquire_leader)
mock_patch.side_effect = k8s_client.rest.ApiException(409, '')
self.assertFalse(self.k.attempt_to_acquire_leader())
def test_take_leader(self):
self.k.take_leader()
self.k._leader_observed_record['leader'] = 'test'
@@ -308,7 +315,7 @@ class TestKubernetesEndpoints(BaseTestKubernetes):
mock_patch.side_effect = k8s_client.rest.ApiException(502, '')
self.assertFalse(self.k.update_leader('123'))
mock_patch.side_effect = RetryFailedError('')
self.assertFalse(self.k.update_leader('123'))
self.assertRaises(KubernetesError, self.k.update_leader, '123')
mock_patch.side_effect = k8s_client.rest.ApiException(409, '')
with patch('time.time', Mock(side_effect=[0, 100, 200, 0, 0, 0, 0, 100, 200])):
self.assertFalse(self.k.update_leader('123'))
@@ -318,6 +325,8 @@ class TestKubernetesEndpoints(BaseTestKubernetes):
mock_read.return_value.metadata.resource_version = '2'
self.assertIsNotNone(self.k._update_leader_with_retry({}, '1', []))
mock_patch.side_effect = k8s_client.rest.ApiException(409, '')
mock_read.side_effect = RetryFailedError('')
self.assertRaises(KubernetesError, self.k.update_leader, '123')
mock_read.side_effect = Exception
self.assertFalse(self.k.update_leader('123'))
tests/test_raft.py
+7 -2
View File
@@ -4,7 +4,7 @@ import tempfile
import time
from mock import Mock, PropertyMock, patch
from patroni.dcs.raft import DynMemberSyncObj, KVStoreTTL, Raft, SyncObjUtility, TCPTransport, _TCPTransport
from patroni.dcs.raft import DynMemberSyncObj, KVStoreTTL, Raft, RaftError, SyncObjUtility, TCPTransport, _TCPTransport
from pysyncobj import SyncObjConf, FAIL_REASON
@@ -79,6 +79,9 @@ class TestKVStoreTTL(unittest.TestCase):
self.assertFalse(self.so.set('foo', 'bar', prevExist=False, ttl=30))
self.assertFalse(self.so.retry(self.so._set, 'foo', {'value': 'buz', 'created': 1, 'updated': 1}, prevValue=''))
self.assertTrue(self.so.retry(self.so._set, 'foo', {'value': 'buz', 'created': 1, 'updated': 1}))
with patch.object(KVStoreTTL, 'retry', Mock(side_effect=RaftError(''))):
self.assertFalse(self.so.set('foo', 'bar'))
self.assertRaises(RaftError, self.so.set, 'foo', 'bar', handle_raft_error=False)
def test_delete(self):
self.so.autoTickPeriod = 0.2
@@ -87,6 +90,8 @@ class TestKVStoreTTL(unittest.TestCase):
self.assertFalse(self.so.delete('foo', prevValue='buz'))
self.assertTrue(self.so.delete('foo', recursive=True))
self.assertFalse(self.so.retry(self.so._delete, 'foo', prevValue=''))
with patch.object(KVStoreTTL, 'retry', Mock(side_effect=RaftError(''))):
self.assertFalse(self.so.delete('foo'))
def test_expire(self):
self.so.set('foo', 'bar', ttl=0.001)
@@ -102,7 +107,7 @@ class TestKVStoreTTL(unittest.TestCase):
callback(True, return_values.pop(0))
with patch('time.time', Mock(side_effect=[1, 100])):
self.assertFalse(self.so.retry(test))
self.assertRaises(RaftError, self.so.retry, test)
self.assertTrue(self.so.retry(test))
self.assertFalse(self.so.retry(test))
tests/test_zookeeper.py
+14 -2
View File
@@ -6,6 +6,7 @@ from kazoo.client import KazooClient, KazooState
from kazoo.exceptions import NoNodeError, NodeExistsError
from kazoo.handlers.threading import SequentialThreadingHandler
from kazoo.protocol.states import KeeperState, ZnodeStat
from kazoo.retry import RetryFailedError
from mock import Mock, PropertyMock, patch
from patroni.dcs.zookeeper import Cluster, Leader, PatroniKazooClient,\
PatroniSequentialThreadingHandler, ZooKeeper, ZooKeeperError
@@ -173,7 +174,6 @@ class TestZooKeeper(unittest.TestCase):
self.zk._inner_load_cluster()
def test_get_cluster(self):
self.assertRaises(ZooKeeperError, self.zk.get_cluster)
cluster = self.zk.get_cluster(True)
self.assertIsInstance(cluster.leader, Leader)
self.zk.status_watcher(None)
@@ -222,13 +222,25 @@ class TestZooKeeper(unittest.TestCase):
self.zk.touch_member({'conn_url': 'postgres://repuser:rep-pass@localhost:5434/postgres',
'api_url': 'http://127.0.0.1:8009/patroni'})
@patch.object(MockKazooClient, 'create', Mock(side_effect=[RetryFailedError, Exception]))
def test_attempt_to_acquire_leader(self):
self.assertRaises(ZooKeeperError, self.zk.attempt_to_acquire_leader)
self.assertFalse(self.zk.attempt_to_acquire_leader())
def test_take_leader(self):
self.zk.take_leader()
with patch.object(MockKazooClient, 'create', Mock(side_effect=Exception)):
self.zk.take_leader()
def test_update_leader(self):
self.assertTrue(self.zk.update_leader(12345))
self.assertFalse(self.zk.update_leader(12345))
with patch.object(MockKazooClient, 'delete', Mock(side_effect=RetryFailedError)):
self.assertRaises(ZooKeeperError, self.zk.update_leader, 12345)
with patch.object(MockKazooClient, 'delete', Mock(side_effect=NoNodeError)):
self.assertTrue(self.zk.update_leader(12345))
with patch.object(MockKazooClient, 'create', Mock(side_effect=[RetryFailedError, Exception])):
self.assertRaises(ZooKeeperError, self.zk.update_leader, 12345)
self.assertFalse(self.zk.update_leader(12345))
@patch.object(Cluster, 'min_version', PropertyMock(return_value=(2, 0)))
def test_write_leader_optime(self):