Quorum based failover (#2668)

To enable quorum commit: ```diff $ patronictl.py edit-config --- +++ @@ -5,3 +5,4 @@ use_pg_rewind: true retry_timeout: 10 ttl: 30 +synchronous_mode: quorum Apply these changes? [y/N]: y Configuration changed ``` By default Patroni will use `ANY 1(list,of,stanbys)` in `synchronous_standby_names`. That is, only one node out of listed replicas will be used for quorum. If you want to increase the number of quorum nodes it is possible to do it with: ```diff $ patronictl edit-config --- +++ @@ -6,3 +6,4 @@ retry_timeout: 10 synchronous_mode: quorum ttl: 30 +synchronous_node_count: 2 Apply these changes? [y/N]: y Configuration changed ``` Good old `synchronous_mode: on` is still supported. Close https://github.com/patroni/patroni/issues/664 Close https://github.com/zalando/patroni/pull/672
2026-08-25 14:53:37 +00:00 · 2024-08-13 08:51:01 +02:00
parent 56dba93c55
commit 384705ad97
23 changed files with 1878 additions and 241 deletions
@@ -18,6 +18,7 @@ from patroni.postgresql.config import ConfigHandler
 from patroni.postgresql.postmaster import PostmasterProcess
 from patroni.postgresql.rewind import Rewind
 from patroni.postgresql.slots import SlotsHandler
+from patroni.postgresql.sync import _SyncState
 from patroni.utils import tzutc
 from patroni.watchdog import Watchdog

@@ -63,7 +64,7 @@ def get_cluster_initialized_without_leader(leader=False, failover=None, sync=Non
                                 'tags': {'clonefrom': True},
                                 'scheduled_restart': {'schedule': "2100-01-01 10:53:07.560445+00:00",
                                                       'postgres_version': '99.0.0'}})
-    syncstate = SyncState(0 if sync else None, sync and sync[0], sync and sync[1])
+    syncstate = SyncState(0 if sync else None, sync and sync[0], sync and sync[1], 0)
    failsafe = {m.name: m.api_url for m in (m1, m2)} if failsafe else None
    return get_cluster(SYSID, leader, [m1, m2], failover, syncstate, cluster_config, failsafe)

@@ -207,6 +208,7 @@ class TestHa(PostgresInit):
    @patch('patroni.dcs.dcs_modules', Mock(return_value=['patroni.dcs.etcd']))
    @patch.object(etcd.Client, 'read', etcd_read)
    @patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379']))
+    @patch.object(Config, '_load_cache', Mock())
    def setUp(self):
        super(TestHa, self).setUp()
        self.p.set_state('running')
@@ -1309,7 +1311,7 @@ class TestHa(PostgresInit):
        self.ha.demote('immediate')
        follow.assert_called_once_with(None)

-    def test_process_sync_replication(self):
+    def test__process_multisync_replication(self):
        self.ha.has_lock = true
        mock_set_sync = self.p.sync_handler.set_synchronous_standby_names = Mock()
        mock_cfg_set_sync = self.p.config.set_synchronous_standby_names = Mock()
@@ -1339,8 +1341,9 @@ class TestHa(PostgresInit):
        self.ha.is_synchronous_mode = true

        # Test sync standby not touched when picking the same node
-        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(['other']),
-                                                               CaseInsensitiveSet(['other'])))
+        self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 1, 1,
+                                                                         CaseInsensitiveSet(['other']),
+                                                                         CaseInsensitiveSet(['other'])))
        self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
        self.ha.run_cycle()
        mock_set_sync.assert_not_called()
@@ -1349,15 +1352,17 @@ class TestHa(PostgresInit):
        mock_cfg_set_sync.reset_mock()

        # Test sync standby is replaced when switching standbys
-        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(['other2']), CaseInsensitiveSet()))
+        self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 0, 0, CaseInsensitiveSet(),
+                                                                         CaseInsensitiveSet(['other2'])))
        self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
        self.ha.run_cycle()
        mock_set_sync.assert_called_once_with(CaseInsensitiveSet(['other2']))
        mock_cfg_set_sync.assert_not_called()

        # Test sync standby is replaced when new standby is joined
-        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(['other2', 'other3']),
-                                                               CaseInsensitiveSet(['other2'])))
+        self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 1, 1,
+                                                                         CaseInsensitiveSet(['other2']),
+                                                                         CaseInsensitiveSet(['other2', 'other3'])))
        self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
        self.ha.run_cycle()
        self.assertEqual(mock_set_sync.call_args_list[0][0], (CaseInsensitiveSet(['other2']),))
@@ -1378,8 +1383,9 @@ class TestHa(PostgresInit):
        self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
        self.ha.dcs.get_cluster = Mock(return_value=get_cluster_initialized_with_leader(sync=('leader', 'other')))
        # self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
-        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(['other2']),
-                                                               CaseInsensitiveSet(['other2'])))
+        self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 1, 1,
+                                                                         CaseInsensitiveSet(['other2']),
+                                                                         CaseInsensitiveSet(['other2'])))
        self.ha.run_cycle()
        self.assertEqual(self.ha.dcs.write_sync_state.call_count, 2)

@@ -1402,9 +1408,10 @@ class TestHa(PostgresInit):
        # Test sync set to '*' when synchronous_mode_strict is enabled
        mock_set_sync.reset_mock()
        mock_cfg_set_sync.reset_mock()
-        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(), CaseInsensitiveSet()))
-        self.ha.cluster.config.data['synchronous_mode_strict'] = True
-        self.ha.run_cycle()
+        self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 0, 0, CaseInsensitiveSet(),
+                                                                         CaseInsensitiveSet()))
+        with patch.object(global_config.__class__, 'is_synchronous_mode_strict', PropertyMock(return_value=True)):
+            self.ha.run_cycle()
        mock_set_sync.assert_called_once_with(CaseInsensitiveSet('*'))
        mock_cfg_set_sync.assert_not_called()

@@ -1432,8 +1439,8 @@ class TestHa(PostgresInit):

        # When we just became primary nobody is sync
        self.assertEqual(self.ha.enforce_primary_role('msg', 'promote msg'), 'promote msg')
-        mock_set_sync.assert_called_once_with(CaseInsensitiveSet())
-        mock_write_sync.assert_called_once_with('leader', None, version=0)
+        mock_set_sync.assert_called_once_with(CaseInsensitiveSet(), 0)
+        mock_write_sync.assert_called_once_with('leader', None, 0, version=0)

        mock_set_sync.reset_mock()

@@ -1471,7 +1478,7 @@ class TestHa(PostgresInit):
        mock_acquire.assert_called_once()
        mock_follow.assert_not_called()
        mock_promote.assert_called_once()
-        mock_write_sync.assert_called_once_with('other', None, version=0)
+        mock_write_sync.assert_called_once_with('other', None, 0, version=0)

    def test_disable_sync_when_restarting(self):
        self.ha.is_synchronous_mode = true
@@ -1513,7 +1520,8 @@ class TestHa(PostgresInit):
        self.ha.is_synchronous_mode = true
        self.ha.has_lock = true
        self.p.name = 'leader'
-        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(), CaseInsensitiveSet()))
+        self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 0, 0,
+                                                                         CaseInsensitiveSet(), CaseInsensitiveSet()))
        self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
        with patch('patroni.ha.logger.info') as mock_logger:
            self.ha.run_cycle()
@@ -1529,7 +1537,8 @@ class TestHa(PostgresInit):
        self.ha.has_lock = true
        self.p.name = 'leader'
        self.ha.cluster = get_cluster_initialized_without_leader(sync=('leader', 'a'))
-        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet('a'), CaseInsensitiveSet()))
+        self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 0, 0,
+                                                                         CaseInsensitiveSet(), CaseInsensitiveSet('a')))
        self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
        mock_set_sync = self.p.sync_handler.set_synchronous_standby_names = Mock()
        with patch('patroni.ha.logger.warning') as mock_logger:
@@ -1699,3 +1708,113 @@ class TestHa(PostgresInit):
            mock_logger.assert_called()
            self.assertTrue(mock_logger.call_args[0][0].startswith('Request to %s coordinator leader'))
            self.assertEqual(mock_logger.call_args[0][1], 'Citus')
+
+    @patch.object(global_config.__class__, 'is_synchronous_mode', PropertyMock(return_value=True))
+    @patch.object(global_config.__class__, 'is_quorum_commit_mode', PropertyMock(return_value=True))
+    def test_process_sync_replication_prepromote(self):
+        self.p._major_version = 90500
+        self.ha.cluster = get_cluster_initialized_without_leader(sync=('other', self.p.name + ',foo'))
+        self.p.is_primary = false
+        self.p.set_role('replica')
+        mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=None)
+        # Postgres 9.5, write_sync_state to DCS failed
+        self.assertEqual(self.ha.run_cycle(),
+                         'Postponing promotion because synchronous replication state was updated by somebody else')
+        self.assertEqual(self.ha.dcs.write_sync_state.call_count, 1)
+        self.assertEqual(mock_write_sync.call_args_list[0][0], (self.p.name, None, 0))
+        self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': 0})
+
+        mock_set_sync = self.p.config.set_synchronous_standby_names = Mock()
+        mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=True)
+        # Postgres 9.5, our name is written to leader of the /sync key, while voters list and ssn is empty
+        self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
+        self.assertEqual(self.ha.dcs.write_sync_state.call_count, 1)
+        self.assertEqual(mock_write_sync.call_args_list[0][0], (self.p.name, None, 0))
+        self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': 0})
+        self.assertEqual(mock_set_sync.call_count, 1)
+        self.assertEqual(mock_set_sync.call_args_list[0][0], (None,))
+
+        self.p._major_version = 90600
+        mock_set_sync.reset_mock()
+        mock_write_sync.reset_mock()
+        self.p.set_role('replica')
+        # Postgres 9.6, with quorum commit we avoid updating /sync key and put some nodes to ssn
+        self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
+        self.assertEqual(mock_write_sync.call_count, 0)
+        self.assertEqual(mock_set_sync.call_count, 1)
+        self.assertEqual(mock_set_sync.call_args_list[0][0], ('2 (foo,other)',))
+
+        self.p._major_version = 150000
+        mock_set_sync.reset_mock()
+        self.p.set_role('replica')
+        self.p.name = 'nonsync'
+        self.ha.fetch_node_status = get_node_status()
+        # Postgres 15, with quorum commit. Non-sync node promoted we avoid updating /sync key and put some nodes to ssn
+        self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
+        self.assertEqual(mock_write_sync.call_count, 0)
+        self.assertEqual(mock_set_sync.call_count, 1)
+        self.assertEqual(mock_set_sync.call_args_list[0][0], ('ANY 3 (foo,other,postgresql0)',))
+
+    @patch.object(global_config.__class__, 'is_synchronous_mode', PropertyMock(return_value=True))
+    @patch.object(global_config.__class__, 'is_quorum_commit_mode', PropertyMock(return_value=True))
+    def test__process_quorum_replication(self):
+        self.p._major_version = 150000
+        self.ha.has_lock = true
+        mock_set_sync = self.p.config.set_synchronous_standby_names = Mock()
+        self.p.name = 'leader'
+
+        mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=None)
+        # Test /sync key is attempted to set and failed when missing or invalid
+        self.p.sync_handler.current_state = Mock(return_value=_SyncState('quorum', 1, 1, CaseInsensitiveSet(['other']),
+                                                                         CaseInsensitiveSet(['other'])))
+        self.ha.run_cycle()
+        self.assertEqual(mock_write_sync.call_count, 1)
+        self.assertEqual(mock_write_sync.call_args_list[0][0], (self.p.name, None, 0))
+        self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': None})
+        self.assertEqual(mock_set_sync.call_count, 0)
+
+        self.ha._promote_timestamp = 1
+        mock_write_sync = self.ha.dcs.write_sync_state = Mock(side_effect=[SyncState(None, self.p.name, None, 0), None])
+        # Test /sync key is attempted to set and succeed when missing or invalid
+        with patch.object(SyncState, 'is_empty', Mock(side_effect=[True, False])):
+            self.ha.run_cycle()
+        self.assertEqual(mock_write_sync.call_count, 2)
+        self.assertEqual(mock_write_sync.call_args_list[0][0], (self.p.name, None, 0))
+        self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': None})
+        self.assertEqual(mock_write_sync.call_args_list[1][0], (self.p.name, CaseInsensitiveSet(['other']), 0))
+        self.assertEqual(mock_write_sync.call_args_list[1][1], {'version': None})
+        self.assertEqual(mock_set_sync.call_count, 0)
+
+        self.p.sync_handler.current_state = Mock(side_effect=[_SyncState('quorum', 1, 0, CaseInsensitiveSet(['foo']),
+                                                                         CaseInsensitiveSet(['other'])),
+                                                              _SyncState('quorum', 1, 1, CaseInsensitiveSet(['foo']),
+                                                                         CaseInsensitiveSet(['foo']))])
+        mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=SyncState(1, 'leader', 'foo', 0))
+        self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'foo'))
+        # Test the sync node is removed from voters, added to ssn
+        with patch.object(Postgresql, 'synchronous_standby_names', Mock(return_value='other')), \
+                patch('time.sleep', Mock()):
+            self.ha.run_cycle()
+        self.assertEqual(mock_write_sync.call_count, 1)
+        self.assertEqual(mock_write_sync.call_args_list[0][0], (self.p.name, CaseInsensitiveSet(), 0))
+        self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': 0})
+        self.assertEqual(mock_set_sync.call_count, 1)
+        self.assertEqual(mock_set_sync.call_args_list[0][0], ('ANY 1 (other)',))
+
+        # Test ANY 1 (*) when synchronous_mode_strict and no nodes available
+        self.p.sync_handler.current_state = Mock(return_value=_SyncState('quorum', 1, 0,
+                                                                         CaseInsensitiveSet(['other', 'foo']),
+                                                                         CaseInsensitiveSet()))
+        mock_write_sync.reset_mock()
+        mock_set_sync.reset_mock()
+        with patch.object(global_config.__class__, 'is_synchronous_mode_strict', PropertyMock(return_value=True)):
+            self.ha.run_cycle()
+        self.assertEqual(mock_write_sync.call_count, 1)
+        self.assertEqual(mock_write_sync.call_args_list[0][0], (self.p.name, CaseInsensitiveSet(), 0))
+        self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': 0})
+        self.assertEqual(mock_set_sync.call_count, 1)
+        self.assertEqual(mock_set_sync.call_args_list[0][0], ('ANY 1 (*)',))
+
+        # Test that _process_quorum_replication doesn't take longer than loop_wait
+        with patch('time.time', Mock(side_effect=[30, 60, 90, 120])):
+            self.ha.process_sync_replication()