mirror of
https://github.com/outbackdingo/patroni.git
synced 2026-08-29 08:59:32 +00:00
Ants AasmaandAlexander Kukushkin
70d718a058
* Only activate watchdog while master and not paused We don't really need the protections while we are not master. This way we only need to tickle the watchdog when we are updating leader key or while demotion is happening. As implemented we might fail to notice to shut down the watchdog if someone demotes postgres and removes leader key behind Patroni's back. There are probably other similar cases. Basically if the administrator if being actively stupid they might get unexpected restarts. That seems fine. * Add configuration change support. Change MODE_REQUIRED to disable leader eligibility instead of closing Patroni. Changes watchdog timeout during the next keepalive when ttl is changed. Watchdog driver and requirement can also be switched online. When watchdog mode is `required` and watchdog setup does not work then the effect is similar to nofailover. Add watchdog_failed to status API to signify this. This is True only when watchdog does not work **AND** it is required. * Reset implementation when config changed while active. * Add watchdog safety margin configuration Defaults to 5 seconds. Basically this is the maximum amount of time that can pass between the calls to odcs.update_leader()` and `watchdog.keepalive()`, which are called right after each other. Should be safe for pretty much any sane scenario and allows the default settings to not trigger watchdog when DCS is not responding. * Cancel bootstrap if watchdog activation fails The system would have demoted itself anyway the next HA loop. Doing it in bootstrap gives at least some other node chance to try bootstrapping in the hope that it is configured correctly. If all nodes are unable to activate they will continue to try until the disk is filled with moved datadirs. Perhaps not ideal behavior, but as the situation is unlikely to resolve itself without administrator intervention it doesn't seem too bad.
75 lines
2.1 KiB
Python
75 lines
2.1 KiB
Python
from behave import step, then
|
|
import time
|
|
|
|
|
|
def polling_loop(timeout, interval=1):
|
|
"""Returns an iterator that returns values until timeout has passed. Timeout is measured from start of iteration."""
|
|
start_time = time.time()
|
|
iteration = 0
|
|
end_time = start_time + timeout
|
|
while time.time() < end_time:
|
|
yield iteration
|
|
iteration += 1
|
|
time.sleep(interval)
|
|
|
|
|
|
@step('I start {name:w} with watchdog')
|
|
def start_patroni_with_watchdog(context, name):
|
|
return context.pctl.start(name, custom_config={'watchdog': True})
|
|
|
|
|
|
@step('{name:w} watchdog has been pinged after {timeout:d} seconds')
|
|
def watchdog_was_pinged(context, name, timeout):
|
|
for _ in polling_loop(timeout):
|
|
if context.pctl.get_watchdog(name).was_pinged:
|
|
return True
|
|
return False
|
|
|
|
|
|
@then('{name:w} watchdog has been closed')
|
|
def watchdog_was_closed(context, name):
|
|
assert context.pctl.get_watchdog(name).was_closed
|
|
|
|
|
|
@step('I wait for next {name:w} watchdog ping')
|
|
def watchdog_reset_pinged(context, name):
|
|
context.pctl.get_watchdog(name).reset()
|
|
|
|
|
|
@then('{name:w} watchdog is triggered after {timeout:d} seconds')
|
|
def watchdog_was_triggered(context, name, timeout):
|
|
for _ in polling_loop(timeout):
|
|
if context.pctl.get_watchdog(name).was_triggered:
|
|
return True
|
|
assert False
|
|
|
|
|
|
@then('{name:w} watchdog was not triggered')
|
|
def watchdog_was_not_triggered(context, name):
|
|
assert not context.pctl.get_watchdog(name).was_triggered
|
|
|
|
|
|
@step('{name:w} checkpoint takes {timeout:d} seconds')
|
|
def checkpoint_hang(context, name, timeout):
|
|
assert context.pctl.checkpoint_hang(name, timeout)
|
|
|
|
|
|
@step('{name:w} hangs for {timeout:d} seconds')
|
|
def patroni_hang(context, name, timeout):
|
|
return context.pctl.patroni_hang(name, timeout)
|
|
|
|
|
|
@step('I terminate {name:w} user processes')
|
|
def terminate_backends(context, name):
|
|
return context.pctl.terminate_backends(name)
|
|
|
|
|
|
@step('Sleep for {timeout:d} seconds')
|
|
def dcs_connection_lost(context, timeout):
|
|
time.sleep(timeout)
|
|
|
|
|
|
@then('{name:w} database is running')
|
|
def database_is_running(context, name):
|
|
assert context.pctl.database_is_running(name)
|