From 1c36112b44b62bb668f0f23f8ad1ff9b1cff5561 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 7 Jul 2023 14:23:04 +0200 Subject: [PATCH] Reduce flakiness of citus behave tests (#2728) * Reduce flakiness of citus behave tests - make a few attempts with timeout when checking registered nodes - get rid from artificial sleep - allow check_registration() function to check secondaries These changes are useful for Quorum based failover (#2668) and future PR that enhances Citus support by registering secondaries in `pg_dist_node`. --- features/citus.feature | 17 ++++++++--------- features/steps/citus.py | 22 +++++++++++++++++----- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/features/citus.feature b/features/citus.feature index 2cf295eb..0ee08fdf 100644 --- a/features/citus.feature +++ b/features/citus.feature @@ -10,20 +10,20 @@ Feature: citus And I start postgres3 in citus group 1 Then replication works from postgres0 to postgres1 after 15 seconds Then replication works from postgres2 to postgres3 after 15 seconds - And postgres0 is registered in the postgres0 as the worker in group 0 - And postgres2 is registered in the postgres0 as the worker in group 1 + And postgres0 is registered in the postgres0 as the primary in group 0 after 5 seconds + And postgres2 is registered in the postgres0 as the primary in group 1 after 5 seconds Scenario: coordinator failover updates pg_dist_node Given I run patronictl.py failover batman --group 0 --candidate postgres1 --force Then postgres1 role is the primary after 10 seconds And replication works from postgres1 to postgres0 after 15 seconds And "sync" key in a group 0 in DCS has sync_standby=postgres0 after 15 seconds - And postgres1 is registered in the postgres2 as the worker in group 0 + And postgres1 is registered in the postgres2 as the primary in group 0 after 5 seconds When I run patronictl.py failover batman --group 0 --candidate postgres0 --force Then postgres0 role is the primary after 10 seconds And replication works from postgres0 to postgres1 after 15 seconds And "sync" key in a group 0 in DCS has sync_standby=postgres1 after 15 seconds - And postgres0 is registered in the postgres2 as the worker in group 0 + And postgres0 is registered in the postgres2 as the primary in group 0 after 5 seconds Scenario: worker switchover doesn't break client queries on the coordinator Given I create a distributed table on postgres0 @@ -33,14 +33,14 @@ Feature: citus And postgres3 role is the primary after 10 seconds And replication works from postgres3 to postgres2 after 15 seconds And "sync" key in a group 1 in DCS has sync_standby=postgres2 after 15 seconds - And postgres3 is registered in the postgres0 as the worker in group 1 + And postgres3 is registered in the postgres0 as the primary in group 1 after 5 seconds And a thread is still alive When I run patronictl.py switchover batman --group 1 --force Then I receive a response returncode 0 And postgres2 role is the primary after 10 seconds And replication works from postgres2 to postgres3 after 15 seconds And "sync" key in a group 1 in DCS has sync_standby=postgres3 after 15 seconds - And postgres2 is registered in the postgres0 as the worker in group 1 + And postgres2 is registered in the postgres0 as the primary in group 1 after 5 seconds And a thread is still alive When I stop a thread Then a distributed table on postgres0 has expected rows @@ -52,7 +52,7 @@ Feature: citus Then I receive a response returncode 0 And postgres2 role is the primary after 10 seconds And replication works from postgres2 to postgres3 after 15 seconds - And postgres2 is registered in the postgres0 as the worker in group 1 + And postgres2 is registered in the postgres0 as the primary in group 1 after 5 seconds And a thread is still alive When I stop a thread Then a distributed table on postgres0 has expected rows @@ -64,8 +64,7 @@ Feature: citus When I run patronictl.py edit-config batman --group 2 -s ttl=20 --force Then I receive a response returncode 0 And I receive a response output "+ttl: 20" - When I sleep for 2 seconds - Then postgres4 is registered in the postgres2 as the worker in group 2 + Then postgres4 is registered in the postgres2 as the primary in group 2 after 5 seconds When I shut down postgres4 Then There is a transaction in progress on postgres0 changing pg_dist_node When I run patronictl.py restart batman postgres2 --group 1 --force diff --git a/features/steps/citus.py b/features/steps/citus.py index d645a504..1af70a30 100644 --- a/features/steps/citus.py +++ b/features/steps/citus.py @@ -44,12 +44,24 @@ def start_citus(context, name, group): return context.pctl.start(name, custom_config={"citus": {"database": "postgres", "group": int(group)}}) -@step('{name1:w} is registered in the {name2:w} as the worker in group {group:d}') -def check_registration(context, name1, name2, group): +@step('{name1:w} is registered in the {name2:w} as the {role:w} in group {group:d} after {time_limit:d} seconds') +def check_registration(context, name1, name2, role, group, time_limit): + time_limit *= context.timeout_multiplier + max_time = time.time() + int(time_limit) + worker_port = int(context.pctl.query(name1, "SHOW port").fetchone()[0]) - r = context.pctl.query(name2, "SELECT nodeport FROM pg_catalog.pg_dist_node WHERE groupid = {0}".format(group)) - assert worker_port == r.fetchone()[0],\ - "Worker {0} is not registered in pg_dist_node on the coordinator {1}".format(name1, name2) + + while time.time() < max_time: + try: + cur = context.pctl.query(name2, "SELECT nodeport, noderole" + " FROM pg_catalog.pg_dist_node WHERE groupid = {0}".format(group)) + mapping = {r[0]: r[1] for r in cur} + if mapping.get(worker_port) == role: + return + except Exception: + pass + time.sleep(1) + assert False, "Node {0} is not registered in pg_dist_node on the node {1}".format(name1, name2) @step('I create a distributed table on {name:w}')