Use cached role as a fallback when postgres is slow (#2082)

In some extreme cases Postgres could be so slow that the normal monitoring query doesn't finish in a few seconds. It results in
the exception being raised from the `Postgresql._cluster_info_state_get()` method, which could lead to the situation that postgres isn't demoted on time.
In order to make it reliable we will catch the exception and use the cached state of postgres (`is_running()` and `role`) to determine whether postgres is running as a primary.

Close https://github.com/zalando/patroni/issues/2073

patroni/postgresql/__init__.py

@@ -369,7 +369,11 @@ class Postgresql(object):
         return self._cluster_info_state_get('received_tli')
 
     def is_leader(self):
-        return bool(self._cluster_info_state_get('timeline'))
+        try:
+            return bool(self._cluster_info_state_get('timeline'))
+        except PostgresConnectionException:
+            logger.warning('Failed to determine PostgreSQL state from the connection, falling back to cached role')
+            return bool(self.is_running() and self.role == 'master')
 
     def replay_paused(self):
         return self._cluster_info_state_get('replay_paused')

tests/test_postgresql.py

@@ -335,7 +335,7 @@ class TestPostgresql(BaseTestPostgresql):
         self.assertTrue(self.p.is_leader())
         self.p.reset_cluster_info_state(None)
         with patch.object(Postgresql, '_query', Mock(side_effect=RetryFailedError(''))):
-            self.assertRaises(PostgresConnectionException, self.p.is_leader)
+            self.assertFalse(self.p.is_leader())
 
     @patch.object(Postgresql, 'controldata', Mock(return_value={'Database cluster state': 'shut down',
                                                                 'Latest checkpoint location': '0/1ADBC18',