diff --git a/features/basic_replication.feature b/features/basic_replication.feature index 140e0cf2..fe766c11 100644 --- a/features/basic_replication.feature +++ b/features/basic_replication.feature @@ -1,11 +1,5 @@ Feature: basic replication - In order to check that basic replication works - As observers - We start the primary and the replica - add a table on the primary - and check that it gets replicated to the replica over time. - We stop the primary and check that the replica promotes itself to primary - We start the old primary and check that it rejoins as a replica. + We should check that the basic bootstrapping, replication and failover works. Scenario: check replication of a single table Given I start postgres0 @@ -14,9 +8,9 @@ Feature: basic replication Then table foo is present on postgres1 after 10 seconds Scenario: check the basic failover - When I shut down postgres0 - Then postgres1 role is the primary after 10 seconds + When I kill postgres0 + Then postgres1 role is the primary after 30 seconds When I start postgres0 Then postgres0 role is the secondary after 10 seconds When I add the table bar to postgres1 - Then table bar is present on postgres1 after 10 seconds \ No newline at end of file + Then table bar is present on postgres0 after 10 seconds diff --git a/features/basic_replication.py b/features/basic_replication.py index 4c26da37..176f1f30 100644 --- a/features/basic_replication.py +++ b/features/basic_replication.py @@ -16,6 +16,18 @@ class BasicReplicationSteps(object): '''I start (\w+)''' return world.pctl.start_patroni(pg_name) + def stop_patroni(self, step, pg_name): + '''I shut down (\w+)''' + return world.pctl.stop_patroni(pg_name) + + def kill_patroni(self, step, pg_name): + '''I kill (\w+)''' + return world.pctl.stop_patroni(pg_name, kill=True) + + def do_sleep(self, step, sleep_seconds): + '''I sleep (\w+)''' + sleep(int(sleep_seconds)) + def add_table(self, step, table_name, pg_name): '''I add the table (\w+) to (\w+)''' # parse the configuration file and get the port @@ -36,7 +48,8 @@ class BasicReplicationSteps(object): def check_role(self, step, pg_name, pg_role, max_promotion_timeout): '''(\w+) role is the (\w+) after (\d+) seconds''' - return world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)) + if not world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)): + assert False, "pg_name role didn't change to {0} after {1} seconds".format(pg_role, max_promotion_timeout) BasicReplicationSteps(world) diff --git a/features/terrain.py b/features/terrain.py index 2a9b23cc..5f66e780 100644 --- a/features/terrain.py +++ b/features/terrain.py @@ -47,9 +47,12 @@ class PatroniController(object): def patroni_is_running(self, pg_name): return pg_name in self.processes and self.processes[pg_name].pid and (self.processes[pg_name].poll() is None) - def stop_patroni(self, pg_name): + def stop_patroni(self, pg_name, kill=False): while self.patroni_is_running(pg_name): - self.processes[pg_name].terminate() + if not kill: + self.processes[pg_name].terminate() + else: + self.processes[pg_name].kill() time.sleep(1) self.log.get('pg_name') and self.log[pg_name].close() del self.processes[pg_name] @@ -142,16 +145,17 @@ class PatroniController(object): def check_role_has_changed_to(self, pg_name, new_role, timeout=10): bound_time = time.time() + timeout - current_role = 't' if new_role == 'primary' else 'f' + recovery_status = False if new_role == 'primary' else True role_has_changed = False while not role_has_changed: cur = self.query(pg_name, "SELECT pg_is_in_recovery()", fail_ok=True) if cur: row = cur.fetchone() - if row and len(row) > 0 and row[0] != current_role: + if row and len(row) > 0 and row[0] == recovery_status: role_has_changed = True if time.time() > bound_time: break + time.sleep(1) return role_has_changed def stop_all(self):