Fix unhandled DCSError during startup phase (#3149)

Ensure DCS connectivity before we check node uniqueness or load dynamic configuration.
This commit is contained in:
Waynerv
2024-09-12 08:55:05 +02:00
committed by GitHub
parent d5d6a51e2c
commit 57ed40f66c
2 changed files with 72 additions and 46 deletions
+33 -18
View File
@@ -15,7 +15,7 @@ import patroni.config as config
from patroni.__main__ import check_psycopg, main as _main, Patroni
from patroni.api import RestApiServer
from patroni.async_executor import AsyncExecutor
from patroni.dcs import Cluster, Member
from patroni.dcs import Cluster, ClusterConfig, Member
from patroni.dcs.etcd import AbstractEtcdClientWithFailover
from patroni.exceptions import DCSError
from patroni.postgresql import Postgresql
@@ -90,11 +90,22 @@ class TestPatroni(unittest.TestCase):
def tearDown(self):
logging.getLogger().handlers[:] = self._handlers
@patch('patroni.dcs.AbstractDCS.get_cluster', Mock(side_effect=[None, DCSError('foo'), None]))
def test_load_dynamic_configuration(self):
def test_apply_dynamic_configuration(self):
empty_cluster = Cluster.empty()
self.p.config._dynamic_configuration = {}
self.p.load_dynamic_configuration()
self.p.load_dynamic_configuration()
self.p.apply_dynamic_configuration(empty_cluster)
self.assertEqual(self.p.config._dynamic_configuration['ttl'], 30)
without_config = empty_cluster._asdict()
del without_config['config']
cluster = Cluster(
config=ClusterConfig(version=1, modify_version=1, data={"ttl": 40}),
**without_config
)
self.p.config._dynamic_configuration = {}
self.p.apply_dynamic_configuration(cluster)
self.assertEqual(self.p.config._dynamic_configuration['ttl'], 40)
@patch('sys.argv', ['patroni.py', 'postgres0.yml'])
@patch('time.sleep', Mock(side_effect=SleepException))
@@ -276,11 +287,9 @@ class TestPatroni(unittest.TestCase):
def test_ensure_unique_name(self):
# None/empty cluster implies unique name
with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=None)):
self.assertIsNone(self.p.ensure_unique_name())
self.assertIsNone(self.p.ensure_unique_name(None))
empty_cluster = Cluster.empty()
with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=empty_cluster)):
self.assertIsNone(self.p.ensure_unique_name())
self.assertIsNone(self.p.ensure_unique_name(empty_cluster))
without_members = empty_cluster._asdict()
del without_members['members']
@@ -289,8 +298,7 @@ class TestPatroni(unittest.TestCase):
members=[Member(version=1, name="distinct", session=1, data={})],
**without_members
)
with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=okay_cluster)):
self.assertIsNone(self.p.ensure_unique_name())
self.assertIsNone(self.p.ensure_unique_name(okay_cluster))
# Cluster with a member with the same name that is running
bad_cluster = Cluster(
@@ -299,10 +307,17 @@ class TestPatroni(unittest.TestCase):
})],
**without_members
)
with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=bad_cluster)):
# If the api of the running node cannot be reached, this implies unique name
with patch('urllib3.PoolManager.request', Mock(side_effect=ConnectionError)):
self.assertIsNone(self.p.ensure_unique_name())
# Only if the api of the running node is reachable do we throw an error
with patch('urllib3.PoolManager.request', Mock()):
self.assertRaises(SystemExit, self.p.ensure_unique_name)
# If the api of the running node cannot be reached, this implies unique name
with patch('urllib3.PoolManager.request', Mock(side_effect=ConnectionError)):
self.assertIsNone(self.p.ensure_unique_name(bad_cluster))
# Only if the api of the running node is reachable do we throw an error
with patch('urllib3.PoolManager.request', Mock()):
self.assertRaises(SystemExit, self.p.ensure_unique_name, bad_cluster)
@patch('patroni.dcs.AbstractDCS.get_cluster', Mock(side_effect=[DCSError('foo'), DCSError('foo'), None]))
def test_ensure_dcs_access(self):
with patch('patroni.__main__.logger.warning') as mock_logger:
result = self.p.ensure_dcs_access()
self.assertEqual(result, None)
self.assertEqual(mock_logger.call_count, 2)