gitops/patroni
1
0
Fork 0
mirror of https://github.com/outbackdingo/patroni.git synced 2026-08-25 14:53:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Permanent physical slots on standby nodes (#2852)

Browse Source 
Create permanent physical replication slots on standby nodes and use `pg_replication_slot_advance()` function to move them forward.

The `restart_lsn` is advanced based on values stored in the `/status` key by the primary node.

When slot is created on a replica it could be ahead the same slot on the primary and therefore there is some period of time when it doesn't protect WAL files from being recycled.
This commit is contained in:
Alexander Kukushkin
2023-09-20 16:50:37 +02:00
committed by GitHub
parent 18d9cb1124
commit bc15813de0
11 changed files with 256 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/__init__.py
+6 -4
View File
@@ -104,12 +104,14 @@ class MockCursor(object):
elif sql.startswith('SELECT slot_name, slot_type, datname, plugin, catalog_xmin'):
self.results = [('ls', 'logical', 'a', 'b', 100, 500, b'123456')]
elif sql.startswith('SELECT slot_name'):
self.results = [('blabla', 'physical'), ('foobar', 'physical'), ('ls', 'logical', 'b', 'a', 5, 100, 500)]
self.results = [('blabla', 'physical', 12345),
('foobar', 'physical', 12345),
('ls', 'logical', 499, 'b', 'a', 5, 100, 500)]
elif sql.startswith('WITH slots AS (SELECT slot_name, active'):
self.results = [(False, True)] if self.rowcount == 1 else []
elif sql.startswith('SELECT CASE WHEN pg_catalog.pg_is_in_recovery()'):
self.results = [(1, 2, 1, 0, False, 1, 1, None, None, 'streaming', '',
[{"slot_name": "ls", "confirmed_flush_lsn": 12345}],
[{"slot_name": "ls", "confirmed_flush_lsn": 12345, "restart_lsn": 12344}],
'on', 'n1', None)]
elif sql.startswith('SELECT pg_catalog.pg_is_in_recovery()'):
self.results = [(False, 2)]
@@ -252,8 +254,8 @@ class BaseTestPostgresql(PostgresInit):
if not os.path.exists(self.p.data_dir):
os.makedirs(self.p.data_dir)
self.leadermem = Member(0, 'leader', 28, {
'state': 'running', 'conn_url': 'postgres://replicator:[email protected]:5435/postgres'})
self.leadermem = Member(0, 'leader', 28, {'xlog_location': 100, 'state': 'running',
'conn_url': 'postgres://replicator:[email protected]:5435/postgres'})
self.leader = Leader(-1, 28, self.leadermem)
self.other = Member(0, 'test-1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres',
'state': 'running', 'tags': {'replicatefrom': 'leader'}})
tests/test_slots.py
+26 -6
View File
@@ -53,6 +53,7 @@ class TestSlotsHandler(BaseTestPostgresql):
self.p.set_role('replica')
with patch.object(Postgresql, 'is_primary', Mock(return_value=False)), \
patch.object(SlotsHandler, 'drop_replication_slot') as mock_drop:
config.data['slots'].pop('ls')
self.s.sync_replication_slots(cluster, False, paused=True)
mock_drop.assert_not_called()
self.p.set_role('primary')
@@ -69,6 +70,8 @@ class TestSlotsHandler(BaseTestPostgresql):
self.assertTrue("test.3" in ca, "non matching {0}".format(ca))
with patch.object(Postgresql, 'major_version', PropertyMock(return_value=90618)):
self.s.sync_replication_slots(cluster, False)
self.p.set_role('replica')
self.s.sync_replication_slots(cluster, False)
def test_cascading_replica_sync_replication_slots(self):
"""Test sync with a cascading replica so physical slots are present on a replica."""
@@ -82,12 +85,12 @@ class TestSlotsHandler(BaseTestPostgresql):
self.p.set_role('replica')
with patch.object(Postgresql, '_query') as mock_query, \
patch.object(Postgresql, 'is_primary', Mock(return_value=False)):
mock_query.return_value = [('ls', 'logical', 'b', 'a', 5, 12345, 105)]
mock_query.return_value = [('ls', 'logical', 104, 'b', 'a', 5, 12345, 105)]
ret = self.s.sync_replication_slots(cluster, False)
self.assertEqual(ret, [])
def test_process_permanent_slots(self):
config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}},
config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}, 'blabla': {'type': 'physical'}},
'ignore_slots': [{'name': 'blabla'}]}, 1)
cluster = Cluster(True, config, self.leader, Status.empty(), [self.me, self.other, self.leadermem],
None, SyncState.empty(), None, None)
@@ -98,8 +101,10 @@ class TestSlotsHandler(BaseTestPostgresql):
mock_query.return_value = [(
1, 0, 0, 0, 0, 0, 0, 0, 0, None, None,
[{"slot_name": "ls", "type": "logical", "datoid": 5, "plugin": "b",
"confirmed_flush_lsn": 12345, "catalog_xmin": 105}])]
self.assertEqual(self.p.slots(), {'ls': 12345})
"confirmed_flush_lsn": 12345, "catalog_xmin": 105, "restart_lsn": 12344},
{"slot_name": "blabla", "type": "physical", "datoid": None, "plugin": None,
"confirmed_flush_lsn": None, "catalog_xmin": 105, "restart_lsn": 12344}])]
self.assertEqual(self.p.slots(), {'ls': 12345, 'blabla': 12344})
self.p.reset_cluster_info_state(None)
mock_query.return_value = [(
@@ -114,8 +119,8 @@ class TestSlotsHandler(BaseTestPostgresql):
self.cluster.slots['ls'] = 12346
with patch.object(SlotsHandler, 'check_logical_slots_readiness', Mock(return_value=False)):
self.assertEqual(self.s.sync_replication_slots(self.cluster, False), [])
self.s._schedule_load_slots = False
with patch.object(MockCursor, 'execute', Mock(side_effect=psycopg.OperationalError)), \
with patch.object(SlotsHandler, '_query', Mock(return_value=[('ls', 'logical', 499, 'b', 'a', 5, 100, 500)])), \
patch.object(MockCursor, 'execute', Mock(side_effect=psycopg.OperationalError)), \
patch.object(SlotsAdvanceThread, 'schedule', Mock(return_value=(True, ['ls']))), \
patch.object(psycopg.OperationalError, 'diag') as mock_diag:
type(mock_diag).sqlstate = PropertyMock(return_value='58P01')
@@ -177,3 +182,18 @@ class TestSlotsHandler(BaseTestPostgresql):
with patch.object(SlotsHandler, 'get_local_connection_cursor', Mock(side_effect=Exception)):
self.s.schedule_advance_slots({'foo': {'bar': 100}})
self.s._advance.sync_slots()
@patch.object(Postgresql, 'is_primary', Mock(return_value=False))
def test_advance_physical_slots(self):
config = ClusterConfig(1, {'slots': {'blabla': {'type': 'physical'}, 'leader': None}}, 1)
cluster = Cluster(True, config, self.leader, Status(0, {'blabla': 12346}),
[self.me, self.other, self.leadermem], None, SyncState.empty(), None, None)
self.s.sync_replication_slots(cluster, False)
with patch.object(SlotsHandler, '_query', Mock(side_effect=[[('blabla', 'physical', 12345, None, None, None,
None, None)], Exception])) as mock_query, \
patch('patroni.postgresql.slots.logger.error') as mock_error:
self.s.sync_replication_slots(cluster, False)
self.assertEqual(mock_query.call_args[0],
("SELECT pg_catalog.pg_replication_slot_advance(%s, %s)", "blabla", '0/303A'))
self.assertEqual(mock_error.call_args[0][0],
"Error while advancing replication slot %s to position '%s': %r")