Handle DCS exceptions when demoting (#2081)

While doing demote due to failure to update leader lock it could happen that DCS goes completely down and the get_cluster() call raise the exception.
Not being properly handled it results in postgres remaining stopped until DCS recovers.

patroni/ha.py

@@ -870,8 +870,11 @@ class Ha(object):
         if mode_control['offline']:
             node_to_follow, leader = None, None
         else:
-            cluster = self.dcs.get_cluster()
-            node_to_follow, leader = self._get_node_to_follow(cluster), cluster.leader
+            try:
+                cluster = self.dcs.get_cluster()
+                node_to_follow, leader = self._get_node_to_follow(cluster), cluster.leader
+            except Exception:
+                node_to_follow, leader = None, None
 
         # FIXME: with mode offline called from DCS exception handler and handle_long_action_in_progress
         # there could be an async action already running, calling follow from here will lead

tests/test_ha.py

@@ -402,6 +402,8 @@ class TestHa(PostgresInit):
         self.ha.has_lock = true
         self.ha.update_lock = false
         self.assertEqual(self.ha.run_cycle(), 'demoted self because failed to update leader lock in DCS')
+        with patch.object(Ha, '_get_node_to_follow', Mock(side_effect=DCSError('foo'))):
+            self.assertEqual(self.ha.run_cycle(), 'demoted self because failed to update leader lock in DCS')
         self.p.is_leader = false
         self.assertEqual(self.ha.run_cycle(), 'not promoting because failed to update leader lock in DCS')