From 7284416ee39d82c6fe7ba32977ec551fa7bfe42e Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 11 May 2023 12:19:25 +0200 Subject: [PATCH] Behave tests --- features/quorum_commit.feature | 68 +++++++++++++++++++++++++++++++++ features/steps/quorum_commit.py | 60 +++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 features/quorum_commit.feature create mode 100644 features/steps/quorum_commit.py diff --git a/features/quorum_commit.feature b/features/quorum_commit.feature new file mode 100644 index 00000000..2204ab59 --- /dev/null +++ b/features/quorum_commit.feature @@ -0,0 +1,68 @@ +Feature: quorum commit + Check basic workfrlows when quorum commit is enabled + + Scenario: check enable quorum commit and that the only leader promotes after restart + Given I start postgres0 + Then postgres0 is a leader after 10 seconds + And there is a non empty initialize key in DCS after 15 seconds + When I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "synchronous_mode": "quorum"} + Then I receive a response code 200 + And sync key in DCS has leader=postgres0 after 20 seconds + And sync key in DCS has quorum=0 after 2 seconds + And synchronous_standby_names on postgres0 is set to "_empty_str_" after 2 seconds + When I shut down postgres0 + And sync key in DCS has leader=postgres0 after 2 seconds + When I start postgres0 + Then postgres0 role is the primary after 10 seconds + When I issue a PATCH request to http://127.0.0.1:8008/config with {"synchronous_mode_strict": true} + Then synchronous_standby_names on postgres0 is set to "ANY 1 (*)" after 10 seconds + + Scenario: check failover with one quorum standby + Given I start postgres1 + Then sync key in DCS has sync_standby=postgres1 after 10 seconds + And synchronous_standby_names on postgres0 is set to "ANY 1 (postgres1)" after 2 seconds + When I shut down postgres0 + Then postgres1 role is the primary after 10 seconds + And sync key in DCS has quorum=0 after 10 seconds + Then synchronous_standby_names on postgres1 is set to "ANY 1 (*)" after 10 seconds + When I start postgres0 + Then sync key in DCS has leader=postgres1 after 10 seconds + Then sync key in DCS has sync_standby=postgres0 after 10 seconds + And synchronous_standby_names on postgres1 is set to "ANY 1 (postgres0)" after 2 seconds + + Scenario: check behavior with three nodes and different replication factor + Given I start postgres2 + Then sync key in DCS has sync_standby=postgres0,postgres2 after 10 seconds + And sync key in DCS has quorum=1 after 2 seconds + And synchronous_standby_names on postgres1 is set to "ANY 1 (postgres0,postgres2)" after 2 seconds + When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_node_count": 2} + Then sync key in DCS has quorum=0 after 10 seconds + And synchronous_standby_names on postgres1 is set to "ANY 2 (postgres0,postgres2)" after 2 seconds + + Scenario: switch from quorum replication to good old multisync and back + Given I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": true, "synchronous_node_count": 1} + And I shut down postgres0 + Then synchronous_standby_names on postgres1 is set to "postgres2" after 10 seconds + And sync key in DCS has sync_standby=postgres2 after 10 seconds + Then sync key in DCS has quorum=0 after 2 seconds + When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": "quorum"} + And I start postgres0 + Then synchronous_standby_names on postgres1 is set to "ANY 1 (postgres0,postgres2)" after 10 seconds + And sync key in DCS has sync_standby=postgres0,postgres2 after 10 seconds + Then sync key in DCS has quorum=1 after 2 seconds + + Scenario: REST API and patronictl + Given I run patronictl.py list batman + Then I receive a response returncode 0 + And I receive a response output "Quorum Standby" + And Status code on GET http://127.0.0.1:8008/quorum is 200 after 3 seconds + And Status code on GET http://127.0.0.1:8010/quorum is 200 after 3 seconds + + Scenario: nosync node is removed from voters and synchronous_standby_names + Given I add tag nosync true to postgres2 config + When I issue an empty POST request to http://127.0.0.1:8010/reload + Then I receive a response code 202 + And sync key in DCS has quorum=0 after 10 seconds + And sync key in DCS has sync_standby=postgres0 after 10 seconds + And synchronous_standby_names on postgres1 is set to "ANY 1 (postgres0)" after 2 seconds + And Status code on GET http://127.0.0.1:8010/quorum is 503 after 10 seconds diff --git a/features/steps/quorum_commit.py b/features/steps/quorum_commit.py new file mode 100644 index 00000000..7cedc0d5 --- /dev/null +++ b/features/steps/quorum_commit.py @@ -0,0 +1,60 @@ +import json +import re +import time + +from behave import step, then + + +@step('sync key in DCS has {key:w}={value} after {time_limit:d} seconds') +def check_sync(context, key, value, time_limit): + time_limit *= context.timeout_multiplier + max_time = time.time() + int(time_limit) + dcs_value = None + while time.time() < max_time: + try: + response = json.loads(context.dcs_ctl.query('sync')) + dcs_value = response.get(key) + if key == 'sync_standby' and set((dcs_value or '').split(',')) == set(value.split(',')): + return + elif str(dcs_value) == value: + return + except Exception: + pass + time.sleep(1) + assert False, "sync does not have {0}={1} (found {2}) in dcs after {3} seconds".format(key, value, + dcs_value, time_limit) + + +@then('synchronous_standby_names on {name:2} is set to "{value}" after {time_limit:d} seconds') +def check_synchronous_standby_names(context, name, value, time_limit): + time_limit *= context.timeout_multiplier + max_time = time.time() + int(time_limit) + + if value == '_empty_str_': + value = '' + + if '(' in value: + m = re.match(r'.*(\d+) \(([^)]+)\)', value) + expected_value = set(m.group(2).split()) + expected_num = m.group(1) + else: + expected_value = set([value]) + expected_num = '1' + + while time.time() < max_time: + try: + ssn = context.pctl.query(name, "SHOW synchronous_standby_names").fetchone()[0] + if '(' in ssn: + m = re.match(r'.*(\d+) \(([^)]+)\)', ssn) + db_value = set(m.group(2).split()) + db_num = m.group(1) + else: + db_value = set([ssn]) + db_num = '1' + if expected_value == db_value and expected_num == db_num: + return + except Exception: + pass + time.sleep(1) + assert False, "synchronous_standby_names is not set to '{0}' (found '{1}') after {2} seconds".format(value, ssn, + time_limit)