diff --git a/patroni/ha.py b/patroni/ha.py index be53d3bc..30a4d039 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -1025,6 +1025,7 @@ class Ha(object): # is data directory empty? if self.state_handler.data_directory_empty(): self.state_handler.set_role('uninitialized') + self.state_handler.stop() # In case datadir went away while we were master. TODO: check for this and try to stop postgresql. self.watchdog.disable() diff --git a/patroni/postgresql.py b/patroni/postgresql.py index edd43f63..8e55224b 100644 --- a/patroni/postgresql.py +++ b/patroni/postgresql.py @@ -71,6 +71,25 @@ def null_context(): yield +def _update_postmaster_info(func): + def wrapper(self): + ret = func(self) + if ret and 'pid' in ret and 'start_time' in ret: + old_pid = self._postmaster_info.get('pid', 0) + old_start_time = self._postmaster_info.get('start_time', 0) + try: + pmpid = int(ret['pid']) + pmstart = int(ret['start_time']) + if pmpid != old_pid or pmstart != old_start_time: # this check removes repeating messages from logs + self._postmaster_info = {'pid': pmpid, 'start_time': pmstart} + logger.info("Updated postmaster info: %s .", self._postmaster_info) + except ValueError: + logger.warning('Cannot update postmaster info with data due garbage in pid file: %s', ret) + return ret + + return wrapper + + class Postgresql(object): # List of parameters which must be always passed to postmaster as command line options @@ -142,6 +161,7 @@ class Postgresql(object): self._pg_hba_conf = os.path.join(self._config_dir, 'pg_hba.conf') self._recovery_conf = os.path.join(self._data_dir, 'recovery.conf') self._postmaster_pid = os.path.join(self._data_dir, 'postmaster.pid') + self._postmaster_info = {'pid': 0, 'start_time': 0} self._trigger_file = config.get('recovery_conf', {}).get('trigger_file') or 'promote' self._trigger_file = os.path.abspath(os.path.join(self._data_dir, self._trigger_file)) @@ -709,6 +729,7 @@ class Postgresql(object): return False return self.is_pid_running(self.get_pid()) + @_update_postmaster_info def read_pid_file(self): """Reads and parses postmaster.pid from the data directory @@ -733,6 +754,25 @@ class Postgresql(object): logger.warning("Garbage pid in postmaster.pid: {0!r}".format(pid)) return 0 + def get_pid_with_lost_data_dir(self): + logger.info("Trying to check if process running without directory " + "with cached postmaster info: %s .", self._postmaster_info) + try: + process = psutil.Process(self._postmaster_info['pid']) + # check difference instead of values because of rounding issues + if abs(self._postmaster_info["start_time"] - process.create_time()) < 2: + return process.pid + else: + logger.info("Process with pid %s was started at different time %s .", + process.pid, process.create_time()) + except psutil.NoSuchProcess: + logger.info("Cannot find process %s .", self._postmaster_info['pid']) + return 0 + + def clean_postmaster_info(self): + self._postmaster_info = {'pid': 0, 'start_time': 0} + logger.info("postmaster info was cleaned.") + @staticmethod def is_pid_running(pid): try: @@ -933,6 +973,11 @@ class Postgresql(object): def _do_stop(self, mode, block_callbacks, checkpoint, on_safepoint): if not self.is_running(): + pid = self.get_pid_with_lost_data_dir() + if pid > 0: + self.terminate_starting_postmaster(pid) + self.clean_postmaster_info() + return True, True if on_safepoint: on_safepoint() return True, False @@ -958,6 +1003,7 @@ class Postgresql(object): on_safepoint() self._wait_for_postmaster_stop(pid) + self.clean_postmaster_info() return True, True