AT for master_start_timeout + minor fixes (#361)

docs/SETTINGS.rst

@@ -14,7 +14,7 @@ Bootstrap configuration
     -  **loop\_wait**: the number of seconds the loop will sleep. Default value: 10
     -  **ttl**: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process. Default value: 30
     -  **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election.
-    -  **master\_start\_timeout**: the amount of time a master is allowed to recover from failures before failover is triggered. Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Best worst case failover time for master failure is: ttl + master\_start\_timeout + ttl, unless master\_start\_timeout is zero, in which case it's just ttl. Set the value according to your durability/availability tradeoff.
+    -  **master\_start\_timeout**: the amount of time a master is allowed to recover from failures before failover is triggered. Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Best worst case failover time for master failure is: loop\_wait + master\_start\_timeout + loop\_wait, unless master\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff.
     -  **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that succesfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See `replication modes documentation <https://github.com/zalando/patroni/blob/master/docs/replication_modes.rst>`__ for details.
     -  **postgresql**:
         -  **use\_pg\_rewind**:whether or not to use pg_rewind

features/basic_replication.feature

@@ -23,13 +23,13 @@ Feature: basic replication
   Scenario: check the basic failover in synchronous mode
     When I kill postgres0
     Then postgres2 role is the primary after 22 seconds
-    When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": null}
+    When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": null, "master_start_timeout": 0}
     Then I receive a response code 200
     When I add the table bar to postgres2
     Then table bar is present on postgres1 after 20 seconds
 
-  Scenario: check the basic failover
-    Given I shut down postgres2
+  Scenario: check immediate failover when master_start_timeout=0
+    Given I kill postmaster on postgres2
     Then postgres1 is a leader after 10 seconds
     And postgres1 role is the primary after 10 seconds
 

features/environment.py

@@ -56,7 +56,7 @@ class AbstractController(object):
             assert False,\
                 "{0} instance is not available for queries after {1} seconds".format(self._name, max_wait_limit)
 
-    def stop(self, kill=False, timeout=15):
+    def stop(self, kill=False, timeout=15, _=False):
         term = False
         start_time = time.time()
 
@@ -112,6 +112,11 @@ class PatroniController(AbstractController):
         return subprocess.Popen(['coverage', 'run', '--source=patroni', '-p', 'patroni.py', self._config],
                                 stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory)
 
+    def stop(self, kill=False, timeout=15, postgres=False):
+        if postgres:
+            return subprocess.call(['pg_ctl', '-D', self._data_dir, 'stop', '-mi', '-w'])
+        super(PatroniController, self).stop(kill, timeout)
+
     def _is_accessible(self):
         return self.query("SELECT 1", fail_ok=True) is not None
 

features/steps/basic_replication.py

@@ -19,6 +19,11 @@ def kill_patroni(context, name):
     return context.pctl.stop(name, kill=True)
 
 
+@step('I kill postmaster on {name:w}')
+def stop_postgres(context, name):
+    return context.pctl.stop(name, postgres=True)
+
+
 @step('I add the table {table_name:w} to {pg_name:w}')
 def add_table(context, table_name, pg_name):
     # parse the configuration file and get the port

patroni/dcs/etcd.py

@@ -226,7 +226,7 @@ class Client(etcd.Client):
         try:
             return [(r.target.to_text(True), r.port) for r in resolver.query(host, 'SRV')]
         except DNSException:
-            logger.exception('Can not resolve SRV for %s', host)
+            logger.warning('Can not resolve SRV for %s', host)
         return []
 
     def _get_machines_cache_from_srv(self, srv):

patroni/ha.py

@@ -25,7 +25,6 @@ class _MemberStatus(namedtuple('_MemberStatus', 'member,reachable,in_recovery,xl
         reachable - `!False` if the node is not reachable or is not responding with correct JSON
         in_recovery - `!True` if pg_is_in_recovery() == true
         xlog_location - value of `replayed_location` or `location` from JSON, dependin on its role.
-        is_lagging - `True` if node considers itself too far behind to promote
         tags - dictionary with values of different tags (i.e. nofailover)
     """
     @classmethod
@@ -801,6 +800,7 @@ class Ha(object):
     def post_recover(self):
         if not self.state_handler.is_running():
             if self.has_lock():
+                self.state_handler.set_role('demoted')
                 self.dcs.delete_leader()
                 self.dcs.reset_cluster()
                 return 'removed leader key after trying and failing to start postgres'

tests/test_etcd.py

@@ -129,7 +129,7 @@ def dns_query(name, _):
 
 def socket_getaddrinfo(*args):
     if args[0] in ('ok', 'localhost', '127.0.0.1'):
-        return [(2, 1, 6, '', ('127.0.0.1', 0)), (10, 1, 6, '', ('::1', 0))]
+        return [(socket.AF_INET, 1, 6, '', ('127.0.0.1', 0)), (socket.AF_INET6, 1, 6, '', ('::1', 0))]
     raise socket.gaierror
 
 
@@ -148,7 +148,7 @@ def http_request(method, url, **kwargs):
 class TestDnsCachingResolver(unittest.TestCase):
 
     @patch('time.sleep', Mock(side_effect=SleepException))
-    @patch('socket.gethostbyname_ex', Mock(side_effect=socket.gaierror))
+    @patch('socket.getaddrinfo', Mock(side_effect=socket.gaierror))
     def test_run(self):
         r = DnsCachingResolver()
         self.assertIsNone(r.resolve_async(''))