Retry if the retry-after http header is set (#1431)

If the K8s API is overwhelmed with requests it might ask to retry.

patroni/dcs/kubernetes.py

@@ -31,6 +31,13 @@ class KubernetesRetriableException(k8s_client.rest.ApiException):
         self.body = orig.body
         self.headers = orig.headers
 
+    @property
+    def sleeptime(self):
+        try:
+            return int(self.headers['retry-after'])
+        except Exception:
+            return None
+
 
 class CoreV1ApiProxy(object):
 
@@ -68,7 +75,7 @@ class CoreV1ApiProxy(object):
             try:
                 return getattr(self._api, func)(*args, **kwargs)
             except k8s_client.rest.ApiException as e:
-                if e.status in (502, 503, 504):  # XXX
+                if e.status in (502, 503, 504) or e.headers and 'retry-after' in e.headers:  # XXX
                     raise KubernetesRetriableException(e)
                 raise
         return wrapper

patroni/utils.py

@@ -334,7 +334,7 @@ class Retry(object):
                     logger.warning('Retry got exception: %s', e)
                     raise RetryFailedError("Too many retry attempts")
                 self._attempts += 1
-                sleeptime = self.sleeptime
+                sleeptime = hasattr(e, 'sleeptime') and e.sleeptime or self.sleeptime
 
                 if self._cur_stoptime is not None and time.time() + sleeptime >= self._cur_stoptime:
                     logger.warning('Retry got exception: %s', e)