Fix bug in kubernetes.update_leader (#1685)

Unhandled exception prevented demoting the primary.
In addition to that wrap the update_leader call in the HA loop into try..except block and implement a test case.

Fixes https://github.com/zalando/patroni/issues/1684

patroni/dcs/kubernetes.py

@@ -901,7 +901,7 @@ class Kubernetes(AbstractDCS):
             else:
                 logger.exception('Permission denied' if e.status == 403 else 'Unexpected error from Kubernetes API')
                 return False
-        except RetryFailedError:
+        except (RetryFailedError, K8sException):
             return False
 
         deadline = retry.stoptime - time.time()

patroni/ha.py

@@ -156,7 +156,11 @@ class Ha(object):
                 last_operation = self.state_handler.last_operation()
             except Exception:
                 logger.exception('Exception when called state_handler.last_operation()')
-        ret = self.dcs.update_leader(last_operation, self._leader_access_is_restricted)
+        try:
+            ret = self.dcs.update_leader(last_operation, self._leader_access_is_restricted)
+        except Exception:
+            logger.exception('Unexpected exception raised from update_leader, please report it as a BUG')
+            ret = False
         self.set_is_leader(ret)
         if ret:
             self.watchdog.keepalive()

tests/test_ha.py

@@ -200,7 +200,8 @@ class TestHa(PostgresInit):
 
     def test_update_lock(self):
         self.p.last_operation = Mock(side_effect=PostgresConnectionException(''))
-        self.assertTrue(self.ha.update_lock(True))
+        self.ha.dcs.update_leader = Mock(side_effect=Exception)
+        self.assertFalse(self.ha.update_lock(True))
 
     @patch.object(Postgresql, 'received_timeline', Mock(return_value=None))
     def test_touch_member(self):