From d138a8db171893cdac749af14f9a0c6c7a2ee2c9 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 9 Dec 2016 12:02:41 +0100 Subject: [PATCH] AT for master_start_timeout + minor fixes (#361) --- docs/SETTINGS.rst | 2 +- features/basic_replication.feature | 6 +++--- features/environment.py | 7 ++++++- features/steps/basic_replication.py | 5 +++++ patroni/dcs/etcd.py | 2 +- patroni/ha.py | 2 +- tests/test_etcd.py | 4 ++-- 7 files changed, 19 insertions(+), 9 deletions(-) diff --git a/docs/SETTINGS.rst b/docs/SETTINGS.rst index dd42ad28..0632bd21 100644 --- a/docs/SETTINGS.rst +++ b/docs/SETTINGS.rst @@ -14,7 +14,7 @@ Bootstrap configuration - **loop\_wait**: the number of seconds the loop will sleep. Default value: 10 - **ttl**: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process. Default value: 30 - **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election. - - **master\_start\_timeout**: the amount of time a master is allowed to recover from failures before failover is triggered. Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Best worst case failover time for master failure is: ttl + master\_start\_timeout + ttl, unless master\_start\_timeout is zero, in which case it's just ttl. Set the value according to your durability/availability tradeoff. + - **master\_start\_timeout**: the amount of time a master is allowed to recover from failures before failover is triggered. Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Best worst case failover time for master failure is: loop\_wait + master\_start\_timeout + loop\_wait, unless master\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff. - **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that succesfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See `replication modes documentation `__ for details. - **postgresql**: - **use\_pg\_rewind**:whether or not to use pg_rewind diff --git a/features/basic_replication.feature b/features/basic_replication.feature index 496c9e1f..539b584a 100644 --- a/features/basic_replication.feature +++ b/features/basic_replication.feature @@ -23,13 +23,13 @@ Feature: basic replication Scenario: check the basic failover in synchronous mode When I kill postgres0 Then postgres2 role is the primary after 22 seconds - When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": null} + When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": null, "master_start_timeout": 0} Then I receive a response code 200 When I add the table bar to postgres2 Then table bar is present on postgres1 after 20 seconds - Scenario: check the basic failover - Given I shut down postgres2 + Scenario: check immediate failover when master_start_timeout=0 + Given I kill postmaster on postgres2 Then postgres1 is a leader after 10 seconds And postgres1 role is the primary after 10 seconds diff --git a/features/environment.py b/features/environment.py index e0e2af58..24de014b 100644 --- a/features/environment.py +++ b/features/environment.py @@ -56,7 +56,7 @@ class AbstractController(object): assert False,\ "{0} instance is not available for queries after {1} seconds".format(self._name, max_wait_limit) - def stop(self, kill=False, timeout=15): + def stop(self, kill=False, timeout=15, _=False): term = False start_time = time.time() @@ -112,6 +112,11 @@ class PatroniController(AbstractController): return subprocess.Popen(['coverage', 'run', '--source=patroni', '-p', 'patroni.py', self._config], stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory) + def stop(self, kill=False, timeout=15, postgres=False): + if postgres: + return subprocess.call(['pg_ctl', '-D', self._data_dir, 'stop', '-mi', '-w']) + super(PatroniController, self).stop(kill, timeout) + def _is_accessible(self): return self.query("SELECT 1", fail_ok=True) is not None diff --git a/features/steps/basic_replication.py b/features/steps/basic_replication.py index 7360d4a0..5d89b5c1 100644 --- a/features/steps/basic_replication.py +++ b/features/steps/basic_replication.py @@ -19,6 +19,11 @@ def kill_patroni(context, name): return context.pctl.stop(name, kill=True) +@step('I kill postmaster on {name:w}') +def stop_postgres(context, name): + return context.pctl.stop(name, postgres=True) + + @step('I add the table {table_name:w} to {pg_name:w}') def add_table(context, table_name, pg_name): # parse the configuration file and get the port diff --git a/patroni/dcs/etcd.py b/patroni/dcs/etcd.py index ade316b9..a09de8e8 100644 --- a/patroni/dcs/etcd.py +++ b/patroni/dcs/etcd.py @@ -226,7 +226,7 @@ class Client(etcd.Client): try: return [(r.target.to_text(True), r.port) for r in resolver.query(host, 'SRV')] except DNSException: - logger.exception('Can not resolve SRV for %s', host) + logger.warning('Can not resolve SRV for %s', host) return [] def _get_machines_cache_from_srv(self, srv): diff --git a/patroni/ha.py b/patroni/ha.py index f17b5331..e7575e71 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -25,7 +25,6 @@ class _MemberStatus(namedtuple('_MemberStatus', 'member,reachable,in_recovery,xl reachable - `!False` if the node is not reachable or is not responding with correct JSON in_recovery - `!True` if pg_is_in_recovery() == true xlog_location - value of `replayed_location` or `location` from JSON, dependin on its role. - is_lagging - `True` if node considers itself too far behind to promote tags - dictionary with values of different tags (i.e. nofailover) """ @classmethod @@ -801,6 +800,7 @@ class Ha(object): def post_recover(self): if not self.state_handler.is_running(): if self.has_lock(): + self.state_handler.set_role('demoted') self.dcs.delete_leader() self.dcs.reset_cluster() return 'removed leader key after trying and failing to start postgres' diff --git a/tests/test_etcd.py b/tests/test_etcd.py index 0be84cb4..e2361b9b 100644 --- a/tests/test_etcd.py +++ b/tests/test_etcd.py @@ -129,7 +129,7 @@ def dns_query(name, _): def socket_getaddrinfo(*args): if args[0] in ('ok', 'localhost', '127.0.0.1'): - return [(2, 1, 6, '', ('127.0.0.1', 0)), (10, 1, 6, '', ('::1', 0))] + return [(socket.AF_INET, 1, 6, '', ('127.0.0.1', 0)), (socket.AF_INET6, 1, 6, '', ('::1', 0))] raise socket.gaierror @@ -148,7 +148,7 @@ def http_request(method, url, **kwargs): class TestDnsCachingResolver(unittest.TestCase): @patch('time.sleep', Mock(side_effect=SleepException)) - @patch('socket.gethostbyname_ex', Mock(side_effect=socket.gaierror)) + @patch('socket.getaddrinfo', Mock(side_effect=socket.gaierror)) def test_run(self): r = DnsCachingResolver() self.assertIsNone(r.resolve_async(''))