From 4daaf2beb0c5e18dfb4cc293c26405ff1d08847c Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 3 Nov 2017 16:22:39 +0100 Subject: [PATCH] Perform crash recovery in a single user mode if postgres died as master (#554) But do it only if pg_rewind is enabled or there is no master at the moment. Such "crash recovery" procedure was advised by Heikki Linnakangas --- patroni/ha.py | 26 ++++++++++++++++++-------- tests/test_ha.py | 14 ++++++++++---- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/patroni/ha.py b/patroni/ha.py index 9a50dfea..be53d3bc 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -59,6 +59,7 @@ class Ha(object): self.old_cluster = None self.recovering = False self._post_bootstrap_task = None + self._crash_recovery_executed = False self._start_timeout = None self._async_executor = AsyncExecutor(self.wakeup) self.watchdog = patroni.watchdog @@ -175,6 +176,11 @@ class Ha(object): self._async_executor.run_async(self.state_handler.rewind, (self.cluster.leader,)) return True + def _start_crash_recovery(self, msg): + self._async_executor.schedule(msg) + self._async_executor.run_async(self.state_handler.fix_cluster_state) + return msg + def recover(self): # Postgres is not running and we will restart in standby mode. Watchdog is not needed until we promote. self.watchdog.disable() @@ -194,6 +200,12 @@ class Ha(object): else: timeout = None + data = self.state_handler.controldata() + if data.get('Database cluster state') == 'in production' and not self._crash_recovery_executed and \ + (self.cluster.is_unlocked() or self.state_handler.can_rewind): + self._crash_recovery_executed = True + return self._start_crash_recovery('doing crash recovery in a single user mode') + self.load_cluster_from_dcs() if self.has_lock(): @@ -208,14 +220,11 @@ class Ha(object): node_to_follow = self._get_node_to_follow(self.cluster) # once we already tried to start postgres but failed, single user mode is a rescue in this case - if self.recovering and not self.state_handler.rewind_executed and self.state_handler.can_rewind: - data = self.state_handler.controldata() - if data.get('Database cluster state') not in ('shut down', 'shut down in recovery'): - self.recovering = False - msg = 'fixing cluster state in a single user mode' - self._async_executor.schedule(msg) - self._async_executor.run_async(self.state_handler.fix_cluster_state) - return msg + if self.recovering and not self.state_handler.rewind_executed \ + and not self._crash_recovery_executed and self.state_handler.can_rewind \ + and data.get('Database cluster state') not in ('shut down', 'shut down in recovery'): + self.recovering = False + return self._start_crash_recovery('fixing cluster state in a single user mode') self.recovering = True @@ -896,6 +905,7 @@ class Ha(object): self.dcs.reset_cluster() return 'removed leader key after trying and failing to start postgres' return 'failed to start postgres' + self._crash_recovery_executed = False return None def cancel_initialization(self): diff --git a/tests/test_ha.py b/tests/test_ha.py index 0484bd36..04d64f81 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -172,21 +172,27 @@ class TestHa(unittest.TestCase): self.assertEquals(self.ha.run_cycle(), 'starting as a secondary') def test_recover_replica_failed(self): - self.p.controldata = lambda: {'Database cluster state': 'in production'} + self.p.controldata = lambda: {'Database cluster state': 'in recovery'} self.p.is_running = false self.p.follow = false self.assertEquals(self.ha.run_cycle(), 'starting as a secondary') self.assertEquals(self.ha.run_cycle(), 'failed to start postgres') - def test_recover_master_failed(self): + def test_recover_former_master(self): self.p.follow = false self.p.is_running = false self.p.name = 'leader' self.p.set_role('master') - self.p.controldata = lambda: {'Database cluster state': 'in production'} + self.p.controldata = lambda: {'Database cluster state': 'shut down'} self.ha.cluster = get_cluster_initialized_with_leader() self.assertEquals(self.ha.run_cycle(), 'starting as readonly because i had the session lock') + @patch.object(Postgresql, 'fix_cluster_state', Mock()) + def test_crash_recovery(self): + self.p.is_running = false + self.p.controldata = lambda: {'Database cluster state': 'in production'} + self.assertEquals(self.ha.run_cycle(), 'doing crash recovery in a single user mode') + @patch.object(Postgresql, 'rewind_needed_and_possible', Mock(return_value=True)) def test_recover_with_rewind(self): self.p.is_running = false @@ -196,7 +202,7 @@ class TestHa(unittest.TestCase): @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True)) @patch.object(Postgresql, 'fix_cluster_state', Mock()) def test_single_user_after_recover_failed(self): - self.p.controldata = lambda: {'Database cluster state': 'in production'} + self.p.controldata = lambda: {'Database cluster state': 'in recovery'} self.p.is_running = false self.p.follow = false self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')