gitops/patroni
1
0
Fork 0
mirror of https://github.com/outbackdingo/patroni.git synced 2026-08-25 14:53:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Configurable retention of members replication slots (#3108)

Browse Source 
Current problem of Patroni that strikes many people is that it removes replication slot for member which key is expired from DCS. As a result, when the replica comes back from a scheduled maintenance WAL segments could be already absent, and it can't continue streaming without pulling files from archive.
With PostgreSQL 16 and newer we get another problem: logical slot on a standby node could be invalidated if physical replication slot on the primary was removed (and `pg_catalog` vacuumed).
The most problematic environment is Kubernetes, where slot is removed nearly instantly when member Pod is deleted.

So far, one of the recommended solutions was to configure permanent physical slots with names that match member names to avoid removal of replication slots. It works, but depending on environment might be non-trivial to implement (when for example members may change their names).

This PR implements support of `member_slots_ttl` global configuration parameter, that controls for how long member replication slots should be kept when the member key is absent. Default value is set to `30min`.
The feature is supported only starting from PostgreSQL 11 and newer, because we want to retain slots not only on the leader node, but on all nodes that could potentially become the new leader, and they should be moved forward using `pg_replication_slot_advance()` function.

One could disable feature and get back to the old behavior by setting `member_slots_ttl` to `0`.
This commit is contained in:
Alexander Kukushkin
2024-08-23 14:50:36 +02:00
committed by GitHub
parent 8c5ab4c07d
commit 6d65aa311a
19 changed files with 227 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_consul.py
+2 -2
View File
@@ -46,8 +46,8 @@ def kv_get(self, key, **kwargs):
'ModifyIndex': 6429, 'Value': b'{"leader": "leader", "sync_standby": null}'},
{'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'failsafe', 'LockIndex': 0,
'ModifyIndex': 6429, 'Value': b'{'},
{'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'status', 'LockIndex': 0,
'ModifyIndex': 6429, 'Value': b'{"optime":4496294792, "slots":{"ls":12345}}'}])
{'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'status', 'LockIndex': 0, 'ModifyIndex': 6429,
'Value': b'{"optime":4496294792,"slots":{"ls":12345},"retain_slots":["postgresql0","postgresql1"]}'}])
if key == 'service/good/':
return good_cls
if key == 'service/broken/':
tests/test_etcd.py
+2 -1
View File
@@ -77,7 +77,8 @@ def etcd_read(self, key, **kwargs):
"modifiedIndex": 20730, "createdIndex": 20730}],
"modifiedIndex": 1581, "createdIndex": 1581},
{"key": "/service/batman5/failsafe", "value": '{', "modifiedIndex": 1582, "createdIndex": 1582},
{"key": "/service/batman5/status", "value": '{"optime":2164261704,"slots":{"ls":12345}}',
{"key": "/service/batman5/status",
"value": '{"optime":2164261704,"slots":{"ls":12345},"retain_slots":["postgresql0","postgresql1"]}',
"modifiedIndex": 1582, "createdIndex": 1582}], "modifiedIndex": 1581, "createdIndex": 1581}}
if key == '/service/legacy/':
response['node']['nodes'].pop()
tests/test_etcd3.py
+2 -1
View File
@@ -225,7 +225,8 @@ class TestEtcd3(BaseTestEtcd3):
"header": {"revision": "1"},
"kvs": [
{"key": base64_encode('/patroni/test/status'),
"value": base64_encode('{"optime":1234567,"slots":{"ls":12345}}'), "mod_revision": '1'}
"value": base64_encode('{"optime":1234567,"slots":{"ls":12345},"retain_slots": ["foo"]}'),
"mod_revision": '1'}
]
})
self.assertIsInstance(self.etcd3.get_cluster(), Cluster)
tests/test_ha.py
+2 -2
View File
@@ -42,8 +42,8 @@ def get_cluster(initialize, leader, members, failover, sync, cluster_config=None
t = datetime.datetime.now().isoformat()
history = TimelineHistory(1, '[[1,67197376,"no recovery target specified","' + t + '","foo"]]',
[(1, 67197376, 'no recovery target specified', t, 'foo')])
cluster_config = cluster_config or ClusterConfig(1, {'check_timeline': True}, 1)
return Cluster(initialize, cluster_config, leader, Status(10, None), members, failover, sync, history, failsafe)
cluster_config = cluster_config or ClusterConfig(1, {'check_timeline': True, 'member_slots_ttl': 0}, 1)
return Cluster(initialize, cluster_config, leader, Status(10, None, []), members, failover, sync, history, failsafe)
def get_cluster_not_initialized_without_leader(cluster_config=None):
tests/test_kubernetes.py
+5 -3
View File
@@ -26,14 +26,16 @@ def mock_list_namespaced_config_map(*args, **kwargs):
'annotations': {'initialize': '123', 'config': '{}'}}
items = [k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))]
metadata.update({'name': 'test-leader',
'annotations': {'optime': '1234x', 'leader': 'p-0', 'ttl': '30s', 'slots': '{', 'failsafe': '{'}})
'annotations': {'optime': '1234x', 'leader': 'p-0', 'ttl': '30s',
'slots': '{', 'retain_slots': '{', 'failsafe': '{'}})
items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
metadata.update({'name': 'test-failover', 'annotations': {'leader': 'p-0'}})
items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
metadata.update({'name': 'test-sync', 'annotations': {'leader': 'p-0'}})
items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
metadata.update({'name': 'test-0-leader', 'labels': {k8s_group_label: '0'},
'annotations': {'optime': '1234x', 'leader': 'p-0', 'ttl': '30s', 'slots': '{', 'failsafe': '{'}})
'annotations': {'optime': '1234x', 'leader': 'p-0', 'ttl': '30s',
'slots': '{', 'retain_slots': '{', 'failsafe': '{'}})
items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
metadata.update({'name': 'test-0-config', 'labels': {k8s_group_label: '0'},
'annotations': {'initialize': '123', 'config': '{}'}})
@@ -421,7 +423,7 @@ class TestKubernetesEndpoints(BaseTestKubernetes):
mock_patch.side_effect = RetryFailedError('')
self.assertRaises(KubernetesError, self.k.update_leader, cluster, '123')
mock_patch.side_effect = k8s_client.rest.ApiException(409, '')
with patch('time.time', Mock(side_effect=[0, 100, 200, 0, 0, 0, 0, 100, 200])):
with patch('time.time', Mock(side_effect=[0, 0, 100, 200, 0, 0, 0, 0, 0, 100, 200])):
self.assertFalse(self.k.update_leader(cluster, '123'))
self.assertFalse(self.k.update_leader(cluster, '123'))
self.assertFalse(self.k.update_leader(cluster, '123'))
tests/test_raft.py
+2 -1
View File
@@ -152,7 +152,8 @@ class TestRaft(unittest.TestCase):
self.assertIsInstance(cluster, Cluster)
self.assertIsInstance(cluster.workers[1], Cluster)
self.assertTrue(raft.delete_leader(cluster.leader))
self.assertTrue(raft._sync_obj.set(raft.status_path, '{"optime":1234567,"slots":{"ls":12345}}'))
self.assertTrue(raft._sync_obj.set(raft.status_path,
'{"optime":1234567,"slots":{"ls":12345},"retain_slots":["postgresql0"]}'))
raft.get_cluster()
self.assertTrue(raft.update_leader(cluster, '1', failsafe={'foo': 'bat'}))
self.assertTrue(raft._sync_obj.set(raft.failsafe_path, '{"foo"}'))
tests/test_slots.py
+19 -14
View File
@@ -38,7 +38,7 @@ class TestSlotsHandler(BaseTestPostgresql):
self.s = self.p.slots_handler
self.p.start()
config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}, 'ls2': None}}, 1)
self.cluster = Cluster(True, config, self.leader, Status(0, {'ls': 12345, 'ls2': 12345}),
self.cluster = Cluster(True, config, self.leader, Status(0, {'ls': 12345, 'ls2': 12345}, []),
[self.me, self.other, self.leadermem], None, SyncState.empty(), None, None)
global_config.update(self.cluster)
self.tags = TestTags()
@@ -47,7 +47,7 @@ class TestSlotsHandler(BaseTestPostgresql):
config = ClusterConfig(1, {'slots': {'test_3': {'database': 'a', 'plugin': 'b'},
'A': 0, 'ls': 0, 'b': {'type': 'logical', 'plugin': '1'}},
'ignore_slots': [{'name': 'blabla'}]}, 1)
cluster = Cluster(True, config, self.leader, Status(0, {'test_3': 10}),
cluster = Cluster(True, config, self.leader, Status(0, {'test_3': 10}, []),
[self.me, self.other, self.leadermem], None, SyncState.empty(), None, None)
global_config.update(cluster)
with mock.patch('patroni.postgresql.Postgresql._query', Mock(side_effect=psycopg.OperationalError)):
@@ -89,7 +89,7 @@ class TestSlotsHandler(BaseTestPostgresql):
'state': 'running', 'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
'tags': {'replicatefrom': 'postgresql0'}
})
cluster = Cluster(True, config, self.leader, Status(0, {'ls': 10}),
cluster = Cluster(True, config, self.leader, Status(0, {'ls': 10}, []),
[self.me, self.other, self.leadermem, cascading_replica], None, SyncState.empty(), None, None)
self.p.set_role('replica')
with patch.object(Postgresql, '_query') as mock_query, \
@@ -114,30 +114,33 @@ class TestSlotsHandler(BaseTestPostgresql):
"confirmed_flush_lsn": 12345, "catalog_xmin": 105, "restart_lsn": 12344},
{"slot_name": "blabla", "type": "physical", "datoid": None, "plugin": None,
"confirmed_flush_lsn": None, "catalog_xmin": 105, "restart_lsn": 12344}])]
self.assertEqual(self.p.slots(), {'ls': 12345, 'blabla': 12344})
self.assertEqual(self.p.slots(), {'ls': 12345, 'blabla': 12344, 'postgresql0': 0})
self.p.reset_cluster_info_state(None)
mock_query.return_value = [(
1, 0, 0, 0, 0, 0, 0, 0, 0, None, None,
[{"slot_name": "ls", "type": "logical", "datoid": 6, "plugin": "b",
"confirmed_flush_lsn": 12345, "catalog_xmin": 105}])]
self.assertEqual(self.p.slots(), {})
self.assertEqual(self.p.slots(), {'postgresql0': 0})
def test_nostream_slot_processing(self):
config = ClusterConfig(
1, {'slots': {'foo': {'type': 'logical', 'database': 'a', 'plugin': 'b'}, 'bar': {'type': 'physical'}}}, 1)
nostream_node = Member(0, 'test-2', 28, {
'state': 'running', 'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
'tags': {'nostream': 'True'}
'tags': {'nostream': 'True'},
'xlog_location': 10,
})
cascade_node = Member(0, 'test-3', 28, {
'state': 'running', 'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
'tags': {'replicatefrom': 'test-2'}
'tags': {'replicatefrom': 'test-2'},
'xlog_location': 98
})
stream_node = Member(0, 'test-4', 28, {
'state': 'running', 'conn_url': 'postgres://replicator:[email protected]:5436/postgres'})
'state': 'running', 'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
'xlog_location': 99})
cluster = Cluster(
True, config, self.leader, Status.empty(),
True, config, self.leader, Status(100, {'leader': 99, 'test_2': 98, 'test_3': 97, 'test_4': 98}, []),
[self.leadermem, nostream_node, cascade_node, stream_node], None, SyncState.empty(), None, None)
global_config.update(cluster)
@@ -147,8 +150,8 @@ class TestSlotsHandler(BaseTestPostgresql):
cluster._get_permanent_slots(self.p, self.leadermem, 'primary'),
{'foo': {'type': 'logical', 'database': 'a', 'plugin': 'b'}, 'bar': {'type': 'physical'}})
self.assertEqual(
cluster._get_members_slots(self.p.name, 'primary'),
{'test_4': {'type': 'physical'}})
cluster._get_members_slots(self.p.name, 'primary', False, True),
{'test_3': {'type': 'physical', 'lsn': 98}, 'test_4': {'type': 'physical', 'lsn': 98}})
# nostream node must not have slot on primary
self.p.name = nostream_node.name
@@ -162,8 +165,10 @@ class TestSlotsHandler(BaseTestPostgresql):
# check cascade member-slot existence on nostream node
self.assertEqual(
cluster._get_members_slots(nostream_node.name, 'replica'),
{'test_3': {'type': 'physical'}})
cluster._get_members_slots(nostream_node.name, 'replica', False, True),
{'leader': {'type': 'physical', 'lsn': 99},
'test_3': {'type': 'physical', 'lsn': 98},
'test_4': {'type': 'physical', 'lsn': 98}})
# cascade also does not entitled to have logical slot on itself ...
self.p.name = cascade_node.name
@@ -291,7 +296,7 @@ class TestSlotsHandler(BaseTestPostgresql):
@patch.object(Postgresql, 'is_primary', Mock(return_value=False))
def test_advance_physical_slots(self):
config = ClusterConfig(1, {'slots': {'blabla': {'type': 'physical'}, 'leader': None}}, 1)
cluster = Cluster(True, config, self.leader, Status(0, {'blabla': 12346}),
cluster = Cluster(True, config, self.leader, Status(0, {'blabla': 12346}, []),
[self.me, self.other, self.leadermem], None, SyncState.empty(), None, None)
global_config.update(cluster)
self.s.sync_replication_slots(cluster, self.tags)
tests/test_zookeeper.py
+2 -1
View File
@@ -54,7 +54,8 @@ class MockKazooClient(Mock):
elif path.endswith('/initialize'):
return (b'foo', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))
elif path.endswith('/status'):
return (b'{"optime":500,"slots":{"ls":1234567}}', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0))
return (b'{"optime":500,"slots":{"ls":1234567},"retain_slots":["postgresql0"]}',
ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0))
elif path.endswith('/failsafe'):
return (b'{a}', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0))
return (b'', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))