shutdown postgresql before bootstrap when we lost data directory (#553)

Tries to kill postgresql before bootstrap to prevent old process from interfering.
Fixes https://github.com/zalando/patroni/issues/542

patroni/ha.py

@@ -1025,6 +1025,7 @@ class Ha(object):
             # is data directory empty?
             if self.state_handler.data_directory_empty():
                 self.state_handler.set_role('uninitialized')
+                self.state_handler.stop()
                 # In case datadir went away while we were master. TODO: check for this and try to stop postgresql.
                 self.watchdog.disable()
 

patroni/postgresql.py

@@ -71,6 +71,25 @@ def null_context():
     yield
 
 
+def _update_postmaster_info(func):
+    def wrapper(self):
+        ret = func(self)
+        if ret and 'pid' in ret and 'start_time' in ret:
+            old_pid = self._postmaster_info.get('pid', 0)
+            old_start_time = self._postmaster_info.get('start_time', 0)
+            try:
+                pmpid = int(ret['pid'])
+                pmstart = int(ret['start_time'])
+                if pmpid != old_pid or pmstart != old_start_time:  # this check removes repeating messages from logs
+                    self._postmaster_info = {'pid': pmpid, 'start_time': pmstart}
+                    logger.info("Updated postmaster info: %s .", self._postmaster_info)
+            except ValueError:
+                logger.warning('Cannot update postmaster info with data due garbage in pid file: %s', ret)
+        return ret
+
+    return wrapper
+
+
 class Postgresql(object):
 
     # List of parameters which must be always passed to postmaster as command line options
@@ -142,6 +161,7 @@ class Postgresql(object):
         self._pg_hba_conf = os.path.join(self._config_dir, 'pg_hba.conf')
         self._recovery_conf = os.path.join(self._data_dir, 'recovery.conf')
         self._postmaster_pid = os.path.join(self._data_dir, 'postmaster.pid')
+        self._postmaster_info = {'pid': 0, 'start_time': 0}
         self._trigger_file = config.get('recovery_conf', {}).get('trigger_file') or 'promote'
         self._trigger_file = os.path.abspath(os.path.join(self._data_dir, self._trigger_file))
 
@@ -709,6 +729,7 @@ class Postgresql(object):
             return False
         return self.is_pid_running(self.get_pid())
 
+    @_update_postmaster_info
     def read_pid_file(self):
         """Reads and parses postmaster.pid from the data directory
 
@@ -733,6 +754,25 @@ class Postgresql(object):
             logger.warning("Garbage pid in postmaster.pid: {0!r}".format(pid))
             return 0
 
+    def get_pid_with_lost_data_dir(self):
+        logger.info("Trying to check if process running without directory "
+                    "with cached postmaster info: %s .", self._postmaster_info)
+        try:
+            process = psutil.Process(self._postmaster_info['pid'])
+            # check difference instead of values because of rounding issues
+            if abs(self._postmaster_info["start_time"] - process.create_time()) < 2:
+                return process.pid
+            else:
+                logger.info("Process with pid %s was started at different time %s .",
+                            process.pid, process.create_time())
+        except psutil.NoSuchProcess:
+            logger.info("Cannot find process %s .", self._postmaster_info['pid'])
+        return 0
+
+    def clean_postmaster_info(self):
+        self._postmaster_info = {'pid': 0, 'start_time': 0}
+        logger.info("postmaster info was cleaned.")
+
     @staticmethod
     def is_pid_running(pid):
         try:
@@ -933,6 +973,11 @@ class Postgresql(object):
 
     def _do_stop(self, mode, block_callbacks, checkpoint, on_safepoint):
         if not self.is_running():
+            pid = self.get_pid_with_lost_data_dir()
+            if pid > 0:
+                self.terminate_starting_postmaster(pid)
+                self.clean_postmaster_info()
+                return True, True
             if on_safepoint:
                 on_safepoint()
             return True, False
@@ -958,6 +1003,7 @@ class Postgresql(object):
             on_safepoint()
 
         self._wait_for_postmaster_stop(pid)
+        self.clean_postmaster_info()
 
         return True, True