mirror of
https://github.com/outbackdingo/patroni.git
synced 2026-08-25 14:53:37 +00:00
Merge branch 'master' of github.com:zalando/patroni into feature/citus-secondaries
This commit is contained in:
Alexander Kukushkin
@@ -6,6 +6,7 @@ if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--datadir", required=True)
|
||||
parser.add_argument("--sourcedir", required=True)
|
||||
parser.add_argument("--test-argument", required=True)
|
||||
args, _ = parser.parse_known_args()
|
||||
|
||||
shutil.copytree(args.sourcedir, args.datadir)
|
||||
|
||||
@@ -82,4 +82,4 @@ Feature: basic replication
|
||||
@reject-duplicate-name
|
||||
Scenario: check graceful rejection when two nodes have the same name
|
||||
Given I start duplicate postgres0 on port 8011
|
||||
Then there is a "Can't start; there is already a node named 'postgres0' running" CRITICAL in the dup-postgres0 patroni log
|
||||
Then there is one of ["Can't start; there is already a node named 'postgres0' running"] CRITICAL in the dup-postgres0 patroni log after 5 seconds
|
||||
|
||||
@@ -75,6 +75,6 @@ Feature: citus
|
||||
And I receive a response output "+ttl: 20"
|
||||
Then postgres4 is registered in the postgres2 as the primary in group 2 after 5 seconds
|
||||
When I shut down postgres4
|
||||
Then There is a transaction in progress on postgres0 changing pg_dist_node
|
||||
Then there is a transaction in progress on postgres0 changing pg_dist_node after 5 seconds
|
||||
When I run patronictl.py restart batman postgres2 --group 1 --force
|
||||
Then a transaction finishes in 20 seconds
|
||||
|
||||
@@ -4,14 +4,14 @@ Feature: dcs failsafe mode
|
||||
Scenario: check failsafe mode can be successfully enabled
|
||||
Given I start postgres0
|
||||
And postgres0 is a leader after 10 seconds
|
||||
And I sleep for 3 seconds
|
||||
When I issue a PATCH request to http://127.0.0.1:8008/config with {"loop_wait": 2, "ttl": 20, "retry_timeout": 5, "failsafe_mode": true}
|
||||
Then "config" key in DCS has ttl=30 after 10 seconds
|
||||
When I issue a PATCH request to http://127.0.0.1:8008/config with {"loop_wait": 2, "ttl": 20, "retry_timeout": 3, "failsafe_mode": true}
|
||||
Then I receive a response code 200
|
||||
And Response on GET http://127.0.0.1:8008/failsafe contains postgres0 after 10 seconds
|
||||
When I issue a GET request to http://127.0.0.1:8008/failsafe
|
||||
Then I receive a response code 200
|
||||
And I receive a response postgres0 http://127.0.0.1:8008/patroni
|
||||
When I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"wal_level": "logical"}}}
|
||||
When I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"wal_level": "logical"}},"slots":{"dcs_slot_1": null,"postgres0":null}}
|
||||
Then I receive a response code 200
|
||||
When I issue a PATCH request to http://127.0.0.1:8008/config with {"slots": {"dcs_slot_0": {"type": "logical", "database": "postgres", "plugin": "test_decoding"}}}
|
||||
Then I receive a response code 200
|
||||
@@ -28,7 +28,6 @@ Feature: dcs failsafe mode
|
||||
When I do a backup of postgres0
|
||||
And I shut down postgres0
|
||||
When I start postgres1 in a cluster batman from backup with no_leader
|
||||
And I sleep for 2 seconds
|
||||
Then postgres1 role is the replica after 12 seconds
|
||||
|
||||
Scenario: check leader and replica are both in /failsafe key after leader is back
|
||||
@@ -45,41 +44,73 @@ Feature: dcs failsafe mode
|
||||
@dcs-failsafe
|
||||
@slot-advance
|
||||
Scenario: check leader and replica are functioning while DCS is down
|
||||
Given logical slot dcs_slot_0 is in sync between postgres0 and postgres1 after 10 seconds
|
||||
Given I get all changes from physical slot dcs_slot_1 on postgres0
|
||||
Then physical slot dcs_slot_1 is in sync between postgres0 and postgres1 after 10 seconds
|
||||
And logical slot dcs_slot_0 is in sync between postgres0 and postgres1 after 10 seconds
|
||||
And DCS is down
|
||||
Then Response on GET http://127.0.0.1:8008/primary contains failsafe_mode_is_active after 12 seconds
|
||||
Then postgres0 role is the primary after 10 seconds
|
||||
And postgres1 role is the replica after 2 seconds
|
||||
And replication works from postgres0 to postgres1 after 10 seconds
|
||||
And I get all changes from logical slot dcs_slot_0 on postgres0
|
||||
And logical slot dcs_slot_0 is in sync between postgres0 and postgres1 after 20 seconds
|
||||
When I get all changes from logical slot dcs_slot_0 on postgres0
|
||||
And I get all changes from physical slot dcs_slot_1 on postgres0
|
||||
Then logical slot dcs_slot_0 is in sync between postgres0 and postgres1 after 20 seconds
|
||||
And physical slot dcs_slot_1 is in sync between postgres0 and postgres1 after 10 seconds
|
||||
|
||||
@dcs-failsafe
|
||||
Scenario: check primary is demoted when one replica is shut down and DCS is down
|
||||
Given DCS is down
|
||||
And I kill postgres1
|
||||
And I kill postmaster on postgres1
|
||||
And I sleep for 2 seconds
|
||||
Then postgres0 role is the replica after 12 seconds
|
||||
|
||||
@dcs-failsafe
|
||||
Scenario: check known replica is promoted when leader is down and DCS is up
|
||||
Given I shut down postgres0
|
||||
Given I kill postgres0
|
||||
And I shut down postmaster on postgres0
|
||||
And DCS is up
|
||||
When I start postgres1
|
||||
Then "members/postgres1" key in DCS has state=running after 10 seconds
|
||||
And postgres1 role is the primary after 25 seconds
|
||||
|
||||
@dcs-failsafe
|
||||
Scenario: check three-node cluster is functioning while DCS is down
|
||||
Scenario: scale to three-node cluster
|
||||
Given I start postgres0
|
||||
And I start postgres2
|
||||
Then "members/postgres2" key in DCS has state=running after 10 seconds
|
||||
And "members/postgres0" key in DCS has state=running after 20 seconds
|
||||
And Response on GET http://127.0.0.1:8008/failsafe contains postgres2 after 10 seconds
|
||||
And replication works from postgres1 to postgres0 after 10 seconds
|
||||
And replication works from postgres1 to postgres2 after 10 seconds
|
||||
|
||||
@dcs-failsafe
|
||||
@slot-advance
|
||||
Scenario: make sure permanent slots exist on replicas
|
||||
Given I issue a PATCH request to http://127.0.0.1:8009/config with {"slots":{"dcs_slot_0":null,"dcs_slot_2":{"type":"logical","database":"postgres","plugin":"test_decoding"}}}
|
||||
Then logical slot dcs_slot_2 is in sync between postgres1 and postgres0 after 20 seconds
|
||||
And logical slot dcs_slot_2 is in sync between postgres1 and postgres2 after 20 seconds
|
||||
When I get all changes from physical slot dcs_slot_1 on postgres1
|
||||
Then physical slot dcs_slot_1 is in sync between postgres1 and postgres0 after 10 seconds
|
||||
And physical slot dcs_slot_1 is in sync between postgres1 and postgres2 after 10 seconds
|
||||
And physical slot postgres0 is in sync between postgres1 and postgres2 after 10 seconds
|
||||
|
||||
@dcs-failsafe
|
||||
Scenario: check three-node cluster is functioning while DCS is down
|
||||
Given DCS is down
|
||||
Then Response on GET http://127.0.0.1:8008/primary contains failsafe_mode_is_active after 12 seconds
|
||||
Then Response on GET http://127.0.0.1:8009/primary contains failsafe_mode_is_active after 12 seconds
|
||||
Then postgres1 role is the primary after 10 seconds
|
||||
And postgres0 role is the replica after 2 seconds
|
||||
And postgres2 role is the replica after 2 seconds
|
||||
|
||||
@dcs-failsafe
|
||||
@slot-advance
|
||||
Scenario: check that permanent slots are in sync between nodes while DCS is down
|
||||
Given replication works from postgres1 to postgres0 after 10 seconds
|
||||
And replication works from postgres1 to postgres2 after 10 seconds
|
||||
When I get all changes from logical slot dcs_slot_2 on postgres1
|
||||
And I get all changes from physical slot dcs_slot_1 on postgres1
|
||||
Then logical slot dcs_slot_2 is in sync between postgres1 and postgres0 after 20 seconds
|
||||
And logical slot dcs_slot_2 is in sync between postgres1 and postgres2 after 20 seconds
|
||||
And physical slot dcs_slot_1 is in sync between postgres1 and postgres0 after 10 seconds
|
||||
And physical slot dcs_slot_1 is in sync between postgres1 and postgres2 after 10 seconds
|
||||
And physical slot postgres0 is in sync between postgres1 and postgres2 after 10 seconds
|
||||
|
||||
+23
-15
@@ -162,9 +162,10 @@ class PatroniController(AbstractController):
|
||||
|
||||
def stop(self, kill=False, timeout=15, postgres=False):
|
||||
if postgres:
|
||||
return subprocess.call(['pg_ctl', '-D', self._data_dir, 'stop', '-mi', '-w'])
|
||||
mode = 'i' if kill else 'f'
|
||||
return subprocess.call(['pg_ctl', '-D', self._data_dir, 'stop', '-m' + mode, '-w'])
|
||||
super(PatroniController, self).stop(kill, timeout)
|
||||
if isinstance(self._context.dcs_ctl, KubernetesController):
|
||||
if isinstance(self._context.dcs_ctl, KubernetesController) and not kill:
|
||||
self._context.dcs_ctl.delete_pod(self._name[8:])
|
||||
if self.watchdog:
|
||||
self.watchdog.stop()
|
||||
@@ -244,6 +245,10 @@ class PatroniController(AbstractController):
|
||||
self.recursive_update(config, custom_config)
|
||||
|
||||
self.recursive_update(config, {
|
||||
'log': {
|
||||
'format': '%(asctime)s %(levelname)s [%(pathname)s:%(lineno)d - %(funcName)s]: %(message)s',
|
||||
'loggers': {'patroni.postgresql.callback_executor': 'DEBUG'}
|
||||
},
|
||||
'bootstrap': {
|
||||
'dcs': {
|
||||
'loop_wait': 2,
|
||||
@@ -649,9 +654,10 @@ class KubernetesController(AbstractExternalDcsController):
|
||||
try:
|
||||
if group is not None:
|
||||
scope = '{0}-{1}'.format(scope, group)
|
||||
ep = scope + {'leader': '', 'history': '-config', 'initialize': '-config'}.get(key, '-' + key)
|
||||
rkey = 'leader' if key in ('status', 'failsafe') else key
|
||||
ep = scope + {'leader': '', 'history': '-config', 'initialize': '-config'}.get(rkey, '-' + rkey)
|
||||
e = self._api.read_namespaced_endpoints(ep, self._namespace)
|
||||
if key != 'sync':
|
||||
if key not in ('sync', 'status', 'failsafe'):
|
||||
return e.metadata.annotations[key]
|
||||
else:
|
||||
return json.dumps(e.metadata.annotations)
|
||||
@@ -687,7 +693,7 @@ class ZooKeeperController(AbstractExternalDcsController):
|
||||
self._client = kazoo.client.KazooClient()
|
||||
|
||||
def process_name(self):
|
||||
return "zookeeper"
|
||||
return "java .*zookeeper"
|
||||
|
||||
def query(self, key, scope='batman', group=None):
|
||||
import kazoo.exceptions
|
||||
@@ -886,22 +892,28 @@ class PatroniPoolController(object):
|
||||
}
|
||||
self.start(to_name, custom_config=custom_config)
|
||||
|
||||
def backup_restore_config(self, params=None):
|
||||
return {
|
||||
'command': (self.BACKUP_RESTORE_SCRIPT
|
||||
+ ' --sourcedir=' + os.path.join(self.patroni_path, 'data', 'basebackup')).replace('\\', '/'),
|
||||
'test-argument': 'test-value', # test config mapping approach on custom bootstrap/replica creation
|
||||
**(params or {}),
|
||||
}
|
||||
|
||||
def bootstrap_from_backup(self, name, cluster_name):
|
||||
custom_config = {
|
||||
'scope': cluster_name,
|
||||
'bootstrap': {
|
||||
'method': 'backup_restore',
|
||||
'backup_restore': {
|
||||
'command': (self.BACKUP_RESTORE_SCRIPT + ' --sourcedir='
|
||||
+ os.path.join(self.patroni_path, 'data', 'basebackup').replace('\\', '/')),
|
||||
'backup_restore': self.backup_restore_config({
|
||||
'recovery_conf': {
|
||||
'recovery_target_action': 'promote',
|
||||
'recovery_target_timeline': 'latest',
|
||||
'restore_command': (self.ARCHIVE_RESTORE_SCRIPT + ' --mode restore '
|
||||
+ '--dirname {} --filename %f --pathname %p').format(
|
||||
os.path.join(self.patroni_path, 'data', 'wal_archive_clone').replace('\\', '/'))
|
||||
}
|
||||
}
|
||||
},
|
||||
})
|
||||
},
|
||||
'postgresql': {
|
||||
'authentication': {
|
||||
@@ -922,11 +934,7 @@ class PatroniPoolController(object):
|
||||
.format(os.path.join(self.patroni_path, 'data', 'wal_archive').replace('\\', '/'))
|
||||
},
|
||||
'create_replica_methods': ['no_leader_bootstrap'],
|
||||
'no_leader_bootstrap': {
|
||||
'command': (self.BACKUP_RESTORE_SCRIPT + ' --sourcedir='
|
||||
+ os.path.join(self.patroni_path, 'data', 'basebackup').replace('\\', '/')),
|
||||
'no_leader': '1'
|
||||
}
|
||||
'no_leader_bootstrap': self.backup_restore_config({'no_leader': '1'})
|
||||
}
|
||||
}
|
||||
self.start(name, custom_config=custom_config)
|
||||
|
||||
@@ -25,10 +25,10 @@ Feature: ignored slots
|
||||
# but Patroni can actually end up dropping them almost immediately, so it's helpful
|
||||
# to verify they exist before we begin testing whether they persist through failover
|
||||
# cycles.
|
||||
Then postgres1 has a logical replication slot named unmanaged_slot_0 with the test_decoding plugin
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_1 with the test_decoding plugin
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_2 with the test_decoding plugin
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_3 with the test_decoding plugin
|
||||
Then postgres1 has a logical replication slot named unmanaged_slot_0 with the test_decoding plugin after 2 seconds
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_1 with the test_decoding plugin after 2 seconds
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_2 with the test_decoding plugin after 2 seconds
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_3 with the test_decoding plugin after 2 seconds
|
||||
|
||||
When I start postgres0
|
||||
Then "members/postgres0" key in DCS has role=replica after 10 seconds
|
||||
@@ -46,16 +46,16 @@ Feature: ignored slots
|
||||
And "members/postgres1" key in DCS has role=replica after 10 seconds
|
||||
# give Patroni time to sync replication slots
|
||||
And I sleep for 2 seconds
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_0 with the test_decoding plugin
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_1 with the test_decoding plugin
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_2 with the test_decoding plugin
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_3 with the test_decoding plugin
|
||||
And postgres1 does not have a logical replication slot named dummy_slot
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_0 with the test_decoding plugin after 2 seconds
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_1 with the test_decoding plugin after 2 seconds
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_2 with the test_decoding plugin after 2 seconds
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_3 with the test_decoding plugin after 2 seconds
|
||||
And postgres1 does not have a replication slot named dummy_slot
|
||||
|
||||
# 3. After a failover the server (now a primary) still has the slot.
|
||||
When I shut down postgres0
|
||||
Then "members/postgres1" key in DCS has role=master after 10 seconds
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_0 with the test_decoding plugin
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_1 with the test_decoding plugin
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_2 with the test_decoding plugin
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_3 with the test_decoding plugin
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_0 with the test_decoding plugin after 2 seconds
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_1 with the test_decoding plugin after 2 seconds
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_2 with the test_decoding plugin after 2 seconds
|
||||
And postgres1 has a logical replication slot named unmanaged_slot_3 with the test_decoding plugin after 2 seconds
|
||||
|
||||
@@ -68,6 +68,7 @@ Scenario: check API requests for the primary-replica pair in the pause mode
|
||||
When I kill postmaster on postgres1
|
||||
And I issue a GET request to http://127.0.0.1:8009/replica
|
||||
Then I receive a response code 503
|
||||
And "members/postgres1" key in DCS has state=stopped after 10 seconds
|
||||
When I run patronictl.py restart batman postgres1 --force
|
||||
Then I receive a response returncode 0
|
||||
Then replication works from postgres0 to postgres1 after 20 seconds
|
||||
@@ -76,7 +77,7 @@ Scenario: check API requests for the primary-replica pair in the pause mode
|
||||
Then I receive a response code 200
|
||||
And I receive a response state running
|
||||
And I receive a response role replica
|
||||
When I run patronictl.py reinit batman postgres1 --force
|
||||
When I run patronictl.py reinit batman postgres1 --force --wait
|
||||
Then I receive a response returncode 0
|
||||
And I receive a response output "Success: reinitialize for member postgres1"
|
||||
And postgres1 role is the secondary after 30 seconds
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
Feature: permanent slots
|
||||
Scenario: check that physical permanent slots are created
|
||||
Given I start postgres0
|
||||
Then postgres0 is a leader after 10 seconds
|
||||
And there is a non empty initialize key in DCS after 15 seconds
|
||||
When I issue a PATCH request to http://127.0.0.1:8008/config with {"slots":{"test_physical":0,"postgres0":0,"postgres1":0,"postgres3":0},"postgresql":{"parameters":{"wal_level":"logical"}}}
|
||||
Then I receive a response code 200
|
||||
And Response on GET http://127.0.0.1:8008/config contains slots after 10 seconds
|
||||
When I start postgres1
|
||||
And I start postgres2
|
||||
And I configure and start postgres3 with a tag replicatefrom postgres2
|
||||
Then postgres0 has a physical replication slot named test_physical after 10 seconds
|
||||
And postgres0 has a physical replication slot named postgres1 after 10 seconds
|
||||
And postgres0 has a physical replication slot named postgres2 after 10 seconds
|
||||
And postgres2 has a physical replication slot named postgres3 after 10 seconds
|
||||
|
||||
@slot-advance
|
||||
Scenario: check that logical permanent slots are created
|
||||
Given I run patronictl.py restart batman postgres0 --force
|
||||
And I issue a PATCH request to http://127.0.0.1:8008/config with {"slots":{"test_logical":{"type":"logical","database":"postgres","plugin":"test_decoding"}}}
|
||||
Then postgres0 has a logical replication slot named test_logical with the test_decoding plugin after 10 seconds
|
||||
|
||||
@slot-advance
|
||||
Scenario: check that permanent slots are created on replicas
|
||||
Given postgres1 has a logical replication slot named test_logical with the test_decoding plugin after 10 seconds
|
||||
Then Logical slot test_logical is in sync between postgres0 and postgres1 after 10 seconds
|
||||
And Logical slot test_logical is in sync between postgres0 and postgres2 after 10 seconds
|
||||
And Logical slot test_logical is in sync between postgres0 and postgres3 after 10 seconds
|
||||
And postgres1 has a physical replication slot named test_physical after 2 seconds
|
||||
And postgres2 has a physical replication slot named test_physical after 2 seconds
|
||||
And postgres3 has a physical replication slot named test_physical after 2 seconds
|
||||
|
||||
@slot-advance
|
||||
Scenario: check permanent physical slots that match with member names
|
||||
Given postgres0 has a physical replication slot named postgres3 after 2 seconds
|
||||
And postgres1 has a physical replication slot named postgres0 after 2 seconds
|
||||
And postgres1 has a physical replication slot named postgres3 after 2 seconds
|
||||
And postgres2 has a physical replication slot named postgres0 after 2 seconds
|
||||
And postgres2 has a physical replication slot named postgres3 after 2 seconds
|
||||
And postgres2 has a physical replication slot named postgres1 after 2 seconds
|
||||
And postgres1 does not have a replication slot named postgres2
|
||||
And postgres3 does not have a replication slot named postgres2
|
||||
|
||||
@slot-advance
|
||||
Scenario: check that permanent slots are advanced on replicas
|
||||
Given I add the table replicate_me to postgres0
|
||||
When I get all changes from logical slot test_logical on postgres0
|
||||
And I get all changes from physical slot test_physical on postgres0
|
||||
Then Logical slot test_logical is in sync between postgres0 and postgres1 after 10 seconds
|
||||
And Physical slot test_physical is in sync between postgres0 and postgres1 after 10 seconds
|
||||
And Logical slot test_logical is in sync between postgres0 and postgres2 after 10 seconds
|
||||
And Physical slot test_physical is in sync between postgres0 and postgres2 after 10 seconds
|
||||
And Logical slot test_logical is in sync between postgres0 and postgres3 after 10 seconds
|
||||
And Physical slot test_physical is in sync between postgres0 and postgres3 after 10 seconds
|
||||
And Physical slot postgres1 is in sync between postgres0 and postgres2 after 10 seconds
|
||||
And Physical slot postgres3 is in sync between postgres2 and postgres0 after 20 seconds
|
||||
And Physical slot postgres3 is in sync between postgres2 and postgres1 after 10 seconds
|
||||
And postgres1 does not have a replication slot named postgres2
|
||||
And postgres3 does not have a replication slot named postgres2
|
||||
|
||||
@slot-advance
|
||||
Scenario: check that only permanent slots are written to the /status key
|
||||
Given "status" key in DCS has test_physical in slots
|
||||
And "status" key in DCS has postgres0 in slots
|
||||
And "status" key in DCS has postgres1 in slots
|
||||
And "status" key in DCS does not have postgres2 in slots
|
||||
And "status" key in DCS has postgres3 in slots
|
||||
|
||||
Scenario: check permanent physical replication slot after failover
|
||||
Given I shut down postgres3
|
||||
And I shut down postgres2
|
||||
And I shut down postgres0
|
||||
Then postgres1 has a physical replication slot named test_physical after 10 seconds
|
||||
And postgres1 has a physical replication slot named postgres0 after 10 seconds
|
||||
And postgres1 has a physical replication slot named postgres3 after 10 seconds
|
||||
@@ -0,0 +1,23 @@
|
||||
Feature: priority replication
|
||||
We should check that we can give nodes priority during failover
|
||||
|
||||
Scenario: check failover priority 0 prevents leaderships
|
||||
Given I configure and start postgres0 with a tag failover_priority 1
|
||||
And I configure and start postgres1 with a tag failover_priority 0
|
||||
Then replication works from postgres0 to postgres1 after 20 seconds
|
||||
When I shut down postgres0
|
||||
And I sleep for 5 seconds
|
||||
Then postgres1 role is the secondary after 10 seconds
|
||||
And there is one of ["following a different leader because I am not allowed to promote"] INFO in the postgres1 patroni log after 5 seconds
|
||||
Given I start postgres0
|
||||
Then postgres0 role is the primary after 10 seconds
|
||||
|
||||
Scenario: check higher failover priority is respected
|
||||
Given I configure and start postgres2 with a tag failover_priority 1
|
||||
And I configure and start postgres3 with a tag failover_priority 2
|
||||
Then replication works from postgres0 to postgres2 after 20 seconds
|
||||
And replication works from postgres0 to postgres3 after 20 seconds
|
||||
When I shut down postgres0
|
||||
And I sleep for 5 seconds
|
||||
Then postgres3 role is the primary after 10 seconds
|
||||
And there is one of ["postgres3 has equally tolerable WAL position and priority 2, while this node has priority 1","Wal position of postgres3 is ahead of my wal position"] INFO in the postgres2 patroni log after 5 seconds
|
||||
@@ -14,6 +14,8 @@ Feature: recovery
|
||||
Then I receive a response code 200
|
||||
And I receive a response role master
|
||||
And I receive a response timeline 1
|
||||
And "members/postgres0" key in DCS has state=running after 12 seconds
|
||||
And replication works from postgres0 to postgres1 after 15 seconds
|
||||
|
||||
Scenario: check immediate failover when master_start_timeout=0
|
||||
Given I issue a PATCH request to http://127.0.0.1:8008/config with {"master_start_timeout": 0}
|
||||
|
||||
@@ -22,9 +22,6 @@ Feature: standby cluster
|
||||
Scenario: check permanent logical slots are synced to the replica
|
||||
Given I run patronictl.py restart batman postgres1 --force
|
||||
Then Logical slot test_logical is in sync between postgres0 and postgres1 after 10 seconds
|
||||
When I add the table replicate_me to postgres1
|
||||
And I get all changes from logical slot test_logical on postgres1
|
||||
Then Logical slot test_logical is in sync between postgres0 and postgres1 after 10 seconds
|
||||
|
||||
Scenario: Detach exiting node from the cluster
|
||||
When I shut down postgres1
|
||||
@@ -54,17 +51,21 @@ Feature: standby cluster
|
||||
When I issue a GET request to http://127.0.0.1:8010/patroni
|
||||
Then I receive a response code 200
|
||||
And I receive a response replication_state streaming
|
||||
And postgres1 does not have a logical replication slot named test_logical
|
||||
And postgres1 does not have a replication slot named test_logical
|
||||
|
||||
Scenario: check switchover
|
||||
Given I run patronictl.py switchover batman1 --force
|
||||
Then Status code on GET http://127.0.0.1:8010/standby_leader is 200 after 10 seconds
|
||||
And postgres1 is replicating from postgres2 after 32 seconds
|
||||
And there is a postgres2_cb.log with "on_start replica batman1\non_role_change standby_leader batman1" in postgres2 data directory
|
||||
|
||||
Scenario: check failover
|
||||
When I kill postgres1
|
||||
And I kill postmaster on postgres1
|
||||
Then postgres2 is replicating from postgres0 after 32 seconds
|
||||
When I issue a GET request to http://127.0.0.1:8010/primary
|
||||
When I kill postgres2
|
||||
And I kill postmaster on postgres2
|
||||
Then postgres1 is replicating from postgres0 after 32 seconds
|
||||
And Status code on GET http://127.0.0.1:8009/standby_leader is 200 after 10 seconds
|
||||
When I issue a GET request to http://127.0.0.1:8009/primary
|
||||
Then I receive a response code 503
|
||||
And I sleep for 3 seconds
|
||||
When I issue a GET request to http://127.0.0.1:8010/standby_leader
|
||||
Then I receive a response code 200
|
||||
And I receive a response role standby_leader
|
||||
And replication works from postgres0 to postgres2 after 15 seconds
|
||||
And there is a postgres2_cb.log with "on_start replica batman1\non_role_change standby_leader batman1" in postgres2 data directory
|
||||
And replication works from postgres0 to postgres1 after 15 seconds
|
||||
And there is a postgres1_cb.log with "on_role_change replica batman1\non_role_change standby_leader batman1" in postgres1 data directory
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import json
|
||||
import patroni.psycopg as pg
|
||||
|
||||
from behave import step, then
|
||||
@@ -35,11 +36,16 @@ def kill_patroni(context, name):
|
||||
return context.pctl.stop(name, kill=True)
|
||||
|
||||
|
||||
@step('I kill postmaster on {name:w}')
|
||||
@step('I shut down postmaster on {name:w}')
|
||||
def stop_postgres(context, name):
|
||||
return context.pctl.stop(name, postgres=True)
|
||||
|
||||
|
||||
@step('I kill postmaster on {name:w}')
|
||||
def kill_postgres(context, name):
|
||||
return context.pctl.stop(name, kill=True, postgres=True)
|
||||
|
||||
|
||||
@step('I add the table {table_name:w} to {pg_name:w}')
|
||||
def add_table(context, table_name, pg_name):
|
||||
# parse the configuration file and get the port
|
||||
@@ -105,11 +111,18 @@ def replication_works(context, primary, replica, time_limit):
|
||||
context.execute_steps(u"""
|
||||
When I add the table test_{0} to {1}
|
||||
Then table test_{0} is present on {2} after {3} seconds
|
||||
""".format(int(time()), primary, replica, time_limit))
|
||||
""".format(str(time()).replace('.', '_').replace(',', '_'), primary, replica, time_limit))
|
||||
|
||||
|
||||
@then('there is a "{message}" {level:w} in the {node} patroni log')
|
||||
def check_patroni_log(context, message, level, node):
|
||||
messsages_of_level = context.pctl.read_patroni_log(node, level)
|
||||
assert any(message in line for line in messsages_of_level), \
|
||||
"There was no {0} {1} in the {2} patroni log".format(message, level, node)
|
||||
@then('there is one of {message_list} {level:w} in the {node} patroni log after {timeout:d} seconds')
|
||||
def check_patroni_log(context, message_list, level, node, timeout):
|
||||
timeout *= context.timeout_multiplier
|
||||
message_list = json.loads(message_list)
|
||||
|
||||
for _ in range(int(timeout)):
|
||||
messsages_of_level = context.pctl.read_patroni_log(node, level)
|
||||
if any(any(message in line for line in messsages_of_level) for message in message_list):
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
assert False, f"There were none of {message_list} {level} in the {node} patroni log after {timeout} seconds"
|
||||
|
||||
@@ -28,7 +28,7 @@ def check_member(context, name, key, value, time_limit):
|
||||
while time.time() < max_time:
|
||||
try:
|
||||
response = json.loads(context.dcs_ctl.query(name))
|
||||
dcs_value = response.get(key)
|
||||
dcs_value = str(response.get(key))
|
||||
if dcs_value == value:
|
||||
return
|
||||
except Exception:
|
||||
|
||||
+13
-7
@@ -115,15 +115,21 @@ def count_rows(context, name):
|
||||
assert rows == context.insert_counter, "Distributed table doesn't have expected amount of rows"
|
||||
|
||||
|
||||
@step("There is a transaction in progress on {name:w} changing pg_dist_node")
|
||||
def check_transaction(context, name):
|
||||
cur = context.pctl.query(name, "SELECT xact_start FROM pg_stat_activity WHERE pid <> pg_backend_pid()"
|
||||
" AND state = 'idle in transaction' AND query ~ 'citus_update_node'")
|
||||
assert cur.rowcount == 1, "There is no idle in transaction updating pg_dist_node"
|
||||
context.xact_start = cur.fetchone()[0]
|
||||
@step("there is a transaction in progress on {name:w} changing pg_dist_node after {time_limit:d} seconds")
|
||||
def check_transaction(context, name, time_limit):
|
||||
time_limit *= context.timeout_multiplier
|
||||
max_time = time.time() + int(time_limit)
|
||||
while time.time() < max_time:
|
||||
cur = context.pctl.query(name, "SELECT xact_start FROM pg_stat_activity WHERE pid <> pg_backend_pid()"
|
||||
" AND state = 'idle in transaction' AND query ~ 'citus_update_node'")
|
||||
if cur.rowcount == 1:
|
||||
context.xact_start = cur.fetchone()[0]
|
||||
return
|
||||
time.sleep(1)
|
||||
assert False, f"There is no idle in transaction on {name} updating pg_dist_node after {time_limit} seconds"
|
||||
|
||||
|
||||
@step("a transaction finishes in {timeout:d} seconds")
|
||||
def check_transaction_timeout(context, timeout):
|
||||
assert (datetime.now(tzutc) - context.xact_start).seconds > timeout, \
|
||||
assert (datetime.now(tzutc) - context.xact_start).seconds >= timeout, \
|
||||
"a transaction finished earlier than in {0} seconds".format(timeout)
|
||||
|
||||
+62
-17
@@ -1,3 +1,4 @@
|
||||
import json
|
||||
import time
|
||||
|
||||
from behave import step, then
|
||||
@@ -15,21 +16,30 @@ def create_logical_replication_slot(context, slot_name, pg_name, plugin):
|
||||
assert False, "Error creating slot {0} on {1} with plugin {2}".format(slot_name, pg_name, plugin)
|
||||
|
||||
|
||||
@then('{pg_name:w} has a logical replication slot named {slot_name} with the {plugin:w} plugin')
|
||||
def has_logical_replication_slot(context, pg_name, slot_name, plugin):
|
||||
try:
|
||||
row = context.pctl.query(pg_name, ("SELECT slot_type, plugin FROM pg_replication_slots"
|
||||
" WHERE slot_name = '{0}'").format(slot_name)).fetchone()
|
||||
assert row, "Couldn't find replication slot named {0}".format(slot_name)
|
||||
assert row[0] == "logical", "Found replication slot named {0} but wasn't a logical slot".format(slot_name)
|
||||
assert row[1] == plugin, ("Found replication slot named {0} but was using plugin "
|
||||
"{1} rather than {2}").format(slot_name, row[1], plugin)
|
||||
except pg.Error:
|
||||
assert False, "Error looking for slot {0} on {1} with plugin {2}".format(slot_name, pg_name, plugin)
|
||||
@step('{pg_name:w} has a logical replication slot named {slot_name}'
|
||||
' with the {plugin:w} plugin after {time_limit:d} seconds')
|
||||
@then('{pg_name:w} has a logical replication slot named {slot_name}'
|
||||
' with the {plugin:w} plugin after {time_limit:d} seconds')
|
||||
def has_logical_replication_slot(context, pg_name, slot_name, plugin, time_limit):
|
||||
time_limit *= context.timeout_multiplier
|
||||
max_time = time.time() + int(time_limit)
|
||||
while time.time() < max_time:
|
||||
try:
|
||||
row = context.pctl.query(pg_name, ("SELECT slot_type, plugin FROM pg_replication_slots"
|
||||
f" WHERE slot_name = '{slot_name}'")).fetchone()
|
||||
if row:
|
||||
assert row[0] == "logical", f"Replication slot {slot_name} isn't a logical but {row[0]}"
|
||||
assert row[1] == plugin, f"Replication slot {slot_name} using plugin {row[1]} rather than {plugin}"
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(1)
|
||||
assert False, f"Error looking for slot {slot_name} on {pg_name} with plugin {plugin}"
|
||||
|
||||
|
||||
@then('{pg_name:w} does not have a logical replication slot named {slot_name}')
|
||||
def does_not_have_logical_replication_slot(context, pg_name, slot_name):
|
||||
@step('{pg_name:w} does not have a replication slot named {slot_name:w}')
|
||||
@then('{pg_name:w} does not have a replication slot named {slot_name:w}')
|
||||
def does_not_have_replication_slot(context, pg_name, slot_name):
|
||||
try:
|
||||
row = context.pctl.query(pg_name, ("SELECT 1 FROM pg_replication_slots"
|
||||
" WHERE slot_name = '{0}'").format(slot_name)).fetchone()
|
||||
@@ -38,13 +48,14 @@ def does_not_have_logical_replication_slot(context, pg_name, slot_name):
|
||||
assert False, "Error looking for slot {0} on {1}".format(slot_name, pg_name)
|
||||
|
||||
|
||||
@step('Logical slot {slot_name:w} is in sync between {pg_name1:w} and {pg_name2:w} after {time_limit:d} seconds')
|
||||
def logical_slots_in_sync(context, slot_name, pg_name1, pg_name2, time_limit):
|
||||
@step('{slot_type:w} slot {slot_name:w} is in sync between {pg_name1:w} and {pg_name2:w} after {time_limit:d} seconds')
|
||||
def slots_in_sync(context, slot_type, slot_name, pg_name1, pg_name2, time_limit):
|
||||
time_limit *= context.timeout_multiplier
|
||||
max_time = time.time() + int(time_limit)
|
||||
column = 'confirmed_flush_lsn' if slot_type.lower() == 'logical' else 'restart_lsn'
|
||||
query = f"SELECT {column} FROM pg_replication_slots WHERE slot_name = '{slot_name}'"
|
||||
while time.time() < max_time:
|
||||
try:
|
||||
query = "SELECT confirmed_flush_lsn FROM pg_replication_slots WHERE slot_name = '{0}'".format(slot_name)
|
||||
slot1 = context.pctl.query(pg_name1, query).fetchone()
|
||||
slot2 = context.pctl.query(pg_name2, query).fetchone()
|
||||
if slot1[0] == slot2[0]:
|
||||
@@ -52,9 +63,43 @@ def logical_slots_in_sync(context, slot_name, pg_name1, pg_name2, time_limit):
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(1)
|
||||
assert False, "Logical slot {0} is not in sync between {1} and {2}".format(slot_name, pg_name1, pg_name2)
|
||||
assert False, \
|
||||
f"{slot_type} slot {slot_name} is not in sync between {pg_name1} and {pg_name2} after {time_limit} seconds"
|
||||
|
||||
|
||||
@step('I get all changes from logical slot {slot_name:w} on {pg_name:w}')
|
||||
def logical_slot_get_changes(context, slot_name, pg_name):
|
||||
context.pctl.query(pg_name, "SELECT * FROM pg_logical_slot_get_changes('{0}', NULL, NULL)".format(slot_name))
|
||||
|
||||
|
||||
@step('I get all changes from physical slot {slot_name:w} on {pg_name:w}')
|
||||
def physical_slot_get_changes(context, slot_name, pg_name):
|
||||
context.pctl.query(pg_name, f"SELECT * FROM pg_replication_slot_advance('{slot_name}', pg_current_wal_lsn())")
|
||||
|
||||
|
||||
@step('{pg_name:w} has a physical replication slot named {slot_name} after {time_limit:d} seconds')
|
||||
def has_physical_replication_slot(context, pg_name, slot_name, time_limit):
|
||||
time_limit *= context.timeout_multiplier
|
||||
max_time = time.time() + int(time_limit)
|
||||
query = f"SELECT * FROM pg_catalog.pg_replication_slots WHERE slot_type = 'physical' AND slot_name = '{slot_name}'"
|
||||
while time.time() < max_time:
|
||||
try:
|
||||
row = context.pctl.query(pg_name, query).fetchone()
|
||||
if row:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(1)
|
||||
assert False, f"Physical slot {slot_name} doesn't exist after {time_limit} seconds"
|
||||
|
||||
|
||||
@step('"{name}" key in DCS has {subkey:w} in {key:w}')
|
||||
def dcs_key_contains(context, name, subkey, key):
|
||||
response = json.loads(context.dcs_ctl.query(name))
|
||||
assert key in response and subkey in response[key], f"{name} key in DCS doesn't have {subkey} in {key}"
|
||||
|
||||
|
||||
@step('"{name}" key in DCS does not have {subkey:w} in {key:w}')
|
||||
def dcs_key_does_not_contain(context, name, subkey, key):
|
||||
response = json.loads(context.dcs_ctl.query(name))
|
||||
assert key not in response or subkey not in response[key], f"{name} key in DCS has {subkey} in {key}"
|
||||
|
||||
@@ -15,9 +15,7 @@ def start_patroni(context, name, cluster_name):
|
||||
"scope": cluster_name,
|
||||
"postgresql": {
|
||||
"callbacks": callbacks(context, name),
|
||||
"backup_restore": {
|
||||
"command": (context.pctl.PYTHON + " features/backup_restore.py --sourcedir="
|
||||
+ os.path.join(context.pctl.patroni_path, 'data', 'basebackup').replace('\\', '/'))}
|
||||
"backup_restore": context.pctl.backup_restore_config()
|
||||
}
|
||||
})
|
||||
|
||||
@@ -34,6 +32,7 @@ def start_patroni_standby_cluster(context, name, cluster_name, name2):
|
||||
"ttl": 20,
|
||||
"loop_wait": 2,
|
||||
"retry_timeout": 5,
|
||||
"synchronous_mode": True, # should be completely ignored
|
||||
"standby_cluster": {
|
||||
"host": "localhost",
|
||||
"port": port,
|
||||
|
||||
Reference in New Issue
Block a user