Refactor Postgresql.query method to use common retry mechanism

query method in an api.py also needs retry in some cases (for example
when we are running is_healthiest_node check).
In all cases we should retry only when connection is closed or broken.
BUT, the connection status must be checked via cursor.connection (old
implementation was using general connection object for that). For
multi-threaded applications this is not appropriate, because some other
thread might restore connection.

In addition to that I've changed most of the unit tests to use `Mock` and
`patch` where it is possible.

tests/test_utils.py

@@ -1,56 +1,34 @@
-import os
-import time
 import unittest
 
-from patroni.exceptions import DCSError
+from mock import Mock, patch
+from patroni.exceptions import PatroniException
 from patroni.utils import Retry, RetryFailedError, reap_children, sigchld_handler, sigterm_handler, sleep
 
 
-def nop(*args, **kwargs):
-    pass
-
-
-def os_waitpid(a, b):
-    return (0, 0)
-
-
 def time_sleep(_):
     sigchld_handler(None, None)
 
 
 class TestUtils(unittest.TestCase):
 
-    def __init__(self, method_name='runTest'):
-        self.setUp = self.set_up
-        self.tearDown = self.tear_down
-        super(TestUtils, self).__init__(method_name)
-
-    def set_up(self):
-        self.time_sleep = time.sleep
-        time.sleep = nop
-
-    def tear_down(self):
-        time.sleep = self.time_sleep
-
     def test_sigterm_handler(self):
         self.assertRaises(SystemExit, sigterm_handler, None, None)
 
+    @patch('time.sleep', Mock())
     def test_reap_children(self):
         reap_children()
-        os.waitpid = os_waitpid
-        sigchld_handler(None, None)
-        reap_children()
+        with patch('os.waitpid', Mock(return_value=(0, 0))):
+            sigchld_handler(None, None)
+            reap_children()
 
+    @patch('time.sleep', time_sleep)
     def test_sleep(self):
-        time.sleep = time_sleep
         sleep(0.01)
 
 
+@patch('time.sleep', Mock())
 class TestRetrySleeper(unittest.TestCase):
 
-    def _pass(self):
-        pass
-
     def _fail(self, times=1):
         scope = dict(times=0)
 
@@ -59,7 +37,7 @@ class TestRetrySleeper(unittest.TestCase):
                 pass
             else:
                 scope['times'] += 1
-                raise DCSError('Failed!')
+                raise PatroniException('Failed!')
         return inner
 
     def _makeOne(self, *args, **kwargs):
@@ -78,20 +56,14 @@ class TestRetrySleeper(unittest.TestCase):
         self.assertEquals(retry._attempts, 1)
 
     def test_maximum_delay(self):
-        def sleep_func(_time):
-            pass
-
-        retry = self._makeOne(delay=10, max_tries=100, sleep_func=sleep_func)
+        retry = self._makeOne(delay=10, max_tries=100)
         retry(self._fail(times=10))
         self.assertTrue(retry._cur_delay < 4000, retry._cur_delay)
         # gevent's sleep function is picky about the type
         self.assertEquals(type(retry._cur_delay), float)
 
     def test_deadline(self):
-        def sleep_func(_time):
-            pass
-
-        retry = self._makeOne(deadline=0.0001, sleep_func=sleep_func)
+        retry = self._makeOne(deadline=0.0001)
         self.assertRaises(RetryFailedError, retry, self._fail(times=100))
 
     def test_copy(self):