Perform crash recovery in a single user mode if postgres died as master (#554)

But do it only if pg_rewind is enabled or there is no master at the moment.
Such "crash recovery" procedure was advised by Heikki Linnakangas

patroni/ha.py

@@ -59,6 +59,7 @@ class Ha(object):
         self.old_cluster = None
         self.recovering = False
         self._post_bootstrap_task = None
+        self._crash_recovery_executed = False
         self._start_timeout = None
         self._async_executor = AsyncExecutor(self.wakeup)
         self.watchdog = patroni.watchdog
@@ -175,6 +176,11 @@ class Ha(object):
             self._async_executor.run_async(self.state_handler.rewind, (self.cluster.leader,))
             return True
 
+    def _start_crash_recovery(self, msg):
+        self._async_executor.schedule(msg)
+        self._async_executor.run_async(self.state_handler.fix_cluster_state)
+        return msg
+
     def recover(self):
         # Postgres is not running and we will restart in standby mode. Watchdog is not needed until we promote.
         self.watchdog.disable()
@@ -194,6 +200,12 @@ class Ha(object):
         else:
             timeout = None
 
+        data = self.state_handler.controldata()
+        if data.get('Database cluster state') == 'in production' and not self._crash_recovery_executed and \
+                (self.cluster.is_unlocked() or self.state_handler.can_rewind):
+            self._crash_recovery_executed = True
+            return self._start_crash_recovery('doing crash recovery in a single user mode')
+
         self.load_cluster_from_dcs()
 
         if self.has_lock():
@@ -208,14 +220,11 @@ class Ha(object):
             node_to_follow = self._get_node_to_follow(self.cluster)
 
         # once we already tried to start postgres but failed, single user mode is a rescue in this case
-        if self.recovering and not self.state_handler.rewind_executed and self.state_handler.can_rewind:
-            data = self.state_handler.controldata()
-            if data.get('Database cluster state') not in ('shut down', 'shut down in recovery'):
-                self.recovering = False
-                msg = 'fixing cluster state in a single user mode'
-                self._async_executor.schedule(msg)
-                self._async_executor.run_async(self.state_handler.fix_cluster_state)
-                return msg
+        if self.recovering and not self.state_handler.rewind_executed \
+                and not self._crash_recovery_executed and self.state_handler.can_rewind \
+                and data.get('Database cluster state') not in ('shut down', 'shut down in recovery'):
+            self.recovering = False
+            return self._start_crash_recovery('fixing cluster state in a single user mode')
 
         self.recovering = True
 
@@ -896,6 +905,7 @@ class Ha(object):
                 self.dcs.reset_cluster()
                 return 'removed leader key after trying and failing to start postgres'
             return 'failed to start postgres'
+        self._crash_recovery_executed = False
         return None
 
     def cancel_initialization(self):

tests/test_ha.py

@@ -172,21 +172,27 @@ class TestHa(unittest.TestCase):
         self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
 
     def test_recover_replica_failed(self):
-        self.p.controldata = lambda: {'Database cluster state': 'in production'}
+        self.p.controldata = lambda: {'Database cluster state': 'in recovery'}
         self.p.is_running = false
         self.p.follow = false
         self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
         self.assertEquals(self.ha.run_cycle(), 'failed to start postgres')
 
-    def test_recover_master_failed(self):
+    def test_recover_former_master(self):
         self.p.follow = false
         self.p.is_running = false
         self.p.name = 'leader'
         self.p.set_role('master')
-        self.p.controldata = lambda: {'Database cluster state': 'in production'}
+        self.p.controldata = lambda: {'Database cluster state': 'shut down'}
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.assertEquals(self.ha.run_cycle(), 'starting as readonly because i had the session lock')
 
+    @patch.object(Postgresql, 'fix_cluster_state', Mock())
+    def test_crash_recovery(self):
+        self.p.is_running = false
+        self.p.controldata = lambda: {'Database cluster state': 'in production'}
+        self.assertEquals(self.ha.run_cycle(), 'doing crash recovery in a single user mode')
+
     @patch.object(Postgresql, 'rewind_needed_and_possible', Mock(return_value=True))
     def test_recover_with_rewind(self):
         self.p.is_running = false
@@ -196,7 +202,7 @@ class TestHa(unittest.TestCase):
     @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
     @patch.object(Postgresql, 'fix_cluster_state', Mock())
     def test_single_user_after_recover_failed(self):
-        self.p.controldata = lambda: {'Database cluster state': 'in production'}
+        self.p.controldata = lambda: {'Database cluster state': 'in recovery'}
         self.p.is_running = false
         self.p.follow = false
         self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')