From d2080a3116a4941f5138b510bc47aa1f2446f790 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 13 Mar 2020 16:36:39 +0100 Subject: [PATCH] Retry if the retry-after http header is set (#1431) If the K8s API is overwhelmed with requests it might ask to retry. --- patroni/dcs/kubernetes.py | 9 ++++++++- patroni/utils.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/patroni/dcs/kubernetes.py b/patroni/dcs/kubernetes.py index 6c388102..9244ca04 100644 --- a/patroni/dcs/kubernetes.py +++ b/patroni/dcs/kubernetes.py @@ -31,6 +31,13 @@ class KubernetesRetriableException(k8s_client.rest.ApiException): self.body = orig.body self.headers = orig.headers + @property + def sleeptime(self): + try: + return int(self.headers['retry-after']) + except Exception: + return None + class CoreV1ApiProxy(object): @@ -68,7 +75,7 @@ class CoreV1ApiProxy(object): try: return getattr(self._api, func)(*args, **kwargs) except k8s_client.rest.ApiException as e: - if e.status in (502, 503, 504): # XXX + if e.status in (502, 503, 504) or e.headers and 'retry-after' in e.headers: # XXX raise KubernetesRetriableException(e) raise return wrapper diff --git a/patroni/utils.py b/patroni/utils.py index 2d35963e..6ac3a5a9 100644 --- a/patroni/utils.py +++ b/patroni/utils.py @@ -334,7 +334,7 @@ class Retry(object): logger.warning('Retry got exception: %s', e) raise RetryFailedError("Too many retry attempts") self._attempts += 1 - sleeptime = self.sleeptime + sleeptime = hasattr(e, 'sleeptime') and e.sleeptime or self.sleeptime if self._cur_stoptime is not None and time.time() + sleeptime >= self._cur_stoptime: logger.warning('Retry got exception: %s', e)