Handle exceptions raised from psutil (#610)

Process.cmdline can raise `NoSuchProcess` or `AccessDenied`
Process.children can raise `AccessDenied`

Fixes https://github.com/zalando/patroni/issues/609

patroni/postmaster.py

@@ -82,11 +82,23 @@ class PostmasterProcess(psutil.Process):
                                  "autovacuum launcher|autovacuum worker|stats collector|wal receiver|archiver|"
                                  "wal sender) process|bgworker: )")
 
-        user_backends = [p for p in self.children() if not aux_proc_re.match(p.cmdline()[0])]
-        logger.debug("Waiting for user backends {0} to close".format(
-            ",".join(p.cmdline()[0] for p in user_backends)))
-        psutil.wait_procs(user_backends)
-        logger.debug("Backends closed")
+        try:
+            user_backends = []
+            user_backends_cmdlines = []
+            for child in self.children():
+                try:
+                    cmdline = child.cmdline()[0]
+                    if not aux_proc_re.match(cmdline):
+                        user_backends.append(child)
+                        user_backends_cmdlines.append(cmdline)
+                except psutil.NoSuchProcess:
+                    pass
+            if user_backends:
+                logger.debug('Waiting for user backends %s to close', ', '.join(user_backends_cmdlines))
+                psutil.wait_procs(user_backends)
+            logger.debug("Backends closed")
+        except psutil.Error:
+            logger.exception('wait_for_user_backends_to_close')
 
     @classmethod
     def start(cls, pgcommand, data_dir, conf, options):

tests/test_postmaster.py

@@ -61,11 +61,18 @@ class TestPostmasterProcess(unittest.TestCase):
         c1.cmdline = Mock(return_value=["postgres: startup process"])
         c2 = Mock()
         c2.cmdline = Mock(return_value=["postgres: postgres postgres [local] idle"])
-        with patch('psutil.Process.children', Mock(return_value=[c1, c2])):
+        c3 = Mock()
+        c3.cmdline = Mock(side_effect=psutil.NoSuchProcess(123))
+        with patch('psutil.Process.children', Mock(return_value=[c1, c2, c3])):
             proc = PostmasterProcess(123)
             self.assertIsNone(proc.wait_for_user_backends_to_close())
             mock_wait.assert_called_with([c2])
 
+        c3.cmdline = Mock(side_effect=psutil.AccessDenied(123))
+        with patch('psutil.Process.children', Mock(return_value=[c3])):
+            proc = PostmasterProcess(123)
+            self.assertIsNone(proc.wait_for_user_backends_to_close())
+
     @patch('subprocess.Popen')
     @patch.object(PostmasterProcess, 'from_pid')
     def test_start(self, mock_frompid, mock_popen):