Merge pull request #40 from zalando/feature/ignore-second-sigterm

During shutdown sigterm is sent to all processes every second

patroni/__init__.py

@@ -126,6 +126,7 @@ def main():
     except KeyboardInterrupt:
         pass
     finally:
+        patroni.api.shutdown()
         patroni.touch_member(patroni.shutdown_member_ttl)  # schedule member removal
         patroni.postgresql.stop()
         patroni.ha.dcs.delete_leader()

patroni/etcd.py

@@ -251,7 +251,7 @@ class Etcd(AbstractDCS):
 
             while index and timeout >= 1:  # when timeout is too small urllib3 doesn't have enough time to connect
                 try:
-                    self.client.watch(self.leader_path, index=index + 1, timeout=timeout)
+                    self.client.watch(self.leader_path, index=index + 1, timeout=timeout + 0.5)
                     # Synchronous work of all cluster members with etcd is less expensive
                     # than reestablishing http connection every time from every replica.
                     return True

patroni/utils.py

@@ -8,6 +8,7 @@ import time
 
 from patroni.exceptions import DCSError
 
+ignore_sigterm = False
 interrupted_sleep = False
 reap_children = False
 
@@ -46,7 +47,10 @@ def calculate_ttl(expiration):
 
 
 def sigterm_handler(signo, stack_frame):
-    sys.exit()
+    global ignore_sigterm
+    if not ignore_sigterm:
+        ignore_sigterm = True
+        sys.exit()
 
 
 def sigchld_handler(signo, stack_frame):

tests/test_etcd.py

@@ -67,11 +67,11 @@ def requests_get(url, **kwargs):
 
 
 def etcd_watch(key, index=None, timeout=None, recursive=None):
-    if timeout == 1:
+    if timeout == 2.0:
         raise urllib3.exceptions.TimeoutError
-    elif timeout == 5:
+    elif timeout == 5.0:
         return etcd.EtcdResult('delete', {})
-    elif timeout == 10:
+    elif timeout == 10.0:
         raise etcd.EtcdException
     elif index == 20729:
         return etcd.EtcdResult('set', {'value': 'postgresql1', 'modifiedIndex': index + 1})
@@ -281,7 +281,7 @@ class TestEtcd(unittest.TestCase):
         self.etcd.client.watch = etcd_watch
         self.etcd.watch(100)
         self.etcd.get_cluster()
-        self.etcd.watch(1)
-        self.etcd.watch(5)
-        self.etcd.watch(10)
+        self.etcd.watch(1.5)
+        self.etcd.watch(4.5)
+        self.etcd.watch(9.5)
         self.etcd.watch(100)

tests/test_patroni.py

@@ -40,6 +40,9 @@ def keyboard_interrupt(*args):
 
 class Mock_BaseServer__is_shut_down:
 
+    def wait(self):
+        pass
+
     def set(self):
         pass