gitops/patroni
1
0
Fork 0
mirror of https://github.com/outbackdingo/patroni.git synced 2026-08-25 14:53:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Handle the case when data dir storage disappeared (#2381)

Browse Source 
The `os.listdir()` is raising the OSError exception, breaking the heart-beat loop.

Close https://github.com/zalando/patroni/issues/2380
This commit is contained in:
Alexander Kukushkin
2022-08-15 15:11:59 +02:00
committed by GitHub
parent 2d08e88c3e
commit a0b32379e5
2 changed files with 19 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
patroni/ha.py
+12 -2
View File
@@ -1407,7 +1407,14 @@ class Ha(object):
return 'started as a secondary'
# is data directory empty?
if self.state_handler.data_directory_empty():
try:
data_directory_is_empty = self.state_handler.data_directory_empty()
data_directory_is_accessible = True
except OSError as e:
data_directory_is_accessible = False
data_directory_error = e
if not data_directory_is_accessible or data_directory_is_empty:
self.state_handler.set_role('uninitialized')
self.state_handler.stop('immediate', stop_timeout=self.patroni.config['retry_timeout'])
# In case datadir went away while we were master.
@@ -1416,8 +1423,11 @@ class Ha(object):
# is this instance the leader?
if self.has_lock():
self.release_leader_key_voluntarily()
return 'released leader key voluntarily as data dir empty and currently leader'
return 'released leader key voluntarily as data dir {0} and currently leader'.format(
'empty' if data_directory_is_accessible else 'not accessible')
if not data_directory_is_accessible:
return 'data directory is not accessible: {0}'.format(data_directory_error)
if self.is_paused():
return 'running with empty data directory'
return self.bootstrap() # new node
tests/test_ha.py
+7 -5
View File
@@ -1133,17 +1133,19 @@ class TestHa(PostgresInit):
self.ha.shutdown()
@patch('time.sleep', Mock())
def test_leader_with_empty_directory(self):
def test_leader_with_not_accessible_data_directory(self):
self.ha.cluster = get_cluster_initialized_with_leader()
self.ha.has_lock = true
self.p.data_directory_empty = true
self.assertEqual(self.ha.run_cycle(), 'released leader key voluntarily as data dir empty and currently leader')
self.p.data_directory_empty = Mock(side_effect=OSError(5, "Input/output error: '{}'".format(self.p.data_dir)))
self.assertEqual(self.ha.run_cycle(),
'released leader key voluntarily as data dir not accessible and currently leader')
self.assertEqual(self.p.role, 'uninitialized')
# as has_lock is mocked out, we need to fake the leader key release
self.ha.has_lock = false
# will not say bootstrap from leader as replica can't self elect
self.assertEqual(self.ha.run_cycle(), "trying to bootstrap from replica 'other'")
# will not say bootstrap because data directory is not accessible
self.assertEqual(self.ha.run_cycle(),
"data directory is not accessible: [Errno 5] Input/output error: '{}'".format(self.p.data_dir))
@patch('patroni.postgresql.mtime', Mock(return_value=1588316884))
@patch.object(builtins, 'open', mock_open(read_data=('1\t0/40159C0\tno recovery target specified\n\n'