Feature: failover priority (#2780)

The priority is configured with `failover_priority` tag. Possible values are from `0` till infinity, where `0` means that the node will never become the leader, which is the same as `nofailover` tag set to `true`. As a result, in the configuration file one should set only one of `failover_priority` or `nofailover` tags.

The failover priority kicks in only when there are more than one node have the same receive/replay LSN and are ahead of other nodes in the cluster. In this case the node with higher value of `failover_priority` is preferred. If there is a node with higher values of receive/replay LSN, it will become the new leader even if it has lower value of `failover_priority` (except when priority is set to 0).

Close https://github.com/zalando/patroni/issues/2759

tests/test_config.py

@@ -151,6 +151,53 @@ class TestConfig(unittest.TestCase):
     def test_invalid_path(self):
         self.assertRaises(ConfigParseError, Config, 'postgres0')
 
+    @patch.object(Config, 'get')
+    @patch('patroni.config.logger')
+    def test__validate_failover_tags(self, mock_logger, mock_get):
+        """Ensures that only one of `nofailover` or `failover_priority` can be provided"""
+        mock_logger.warning.reset_mock()
+        config = Config("postgres0.yml")
+        # Providing one of `nofailover` or `failover_priority` is fine
+        just_nofailover = {"nofailover": True}
+        mock_get.side_effect = [just_nofailover] * 2
+        self.assertIsNone(config._validate_failover_tags())
+        mock_logger.warning.assert_not_called()
+        just_failover_priority = {"failover_priority": 1}
+        mock_get.side_effect = [just_failover_priority] * 2
+        self.assertIsNone(config._validate_failover_tags())
+        mock_logger.warning.assert_not_called()
+        # Providing both `nofailover` and `failover_priority` is fine if consistent
+        consistent_false = {"nofailover": False, "failover_priority": 1}
+        mock_get.side_effect = [consistent_false] * 2
+        self.assertIsNone(config._validate_failover_tags())
+        mock_logger.warning.assert_not_called()
+        consistent_true = {"nofailover": True, "failover_priority": 0}
+        mock_get.side_effect = [consistent_true] * 2
+        self.assertIsNone(config._validate_failover_tags())
+        mock_logger.warning.assert_not_called()
+        # Providing both inconsistently should log a warning
+        inconsistent_false = {"nofailover": False, "failover_priority": 0}
+        mock_get.side_effect = [inconsistent_false] * 2
+        self.assertIsNone(config._validate_failover_tags())
+        mock_logger.warning.assert_called_once_with(
+            'Conflicting configuration between nofailover: %s and failover_priority: %s.'
+            + ' Defaulting to nofailover: %s',
+            False,
+            0,
+            False
+        )
+        mock_logger.warning.reset_mock()
+        inconsistent_true = {"nofailover": True, "failover_priority": 1}
+        mock_get.side_effect = [inconsistent_true] * 2
+        self.assertIsNone(config._validate_failover_tags())
+        mock_logger.warning.assert_called_once_with(
+            'Conflicting configuration between nofailover: %s and failover_priority: %s.'
+            + ' Defaulting to nofailover: %s',
+            True,
+            1,
+            True
+        )
+
     def test__process_postgresql_parameters(self):
         expected_params = {
             'f.oo': 'bar',  # not in ConfigHandler.CMDLINE_OPTIONS

tests/test_ha.py

@@ -94,11 +94,12 @@ def get_cluster_initialized_with_leader_and_failsafe():
 
 def get_node_status(reachable=True, in_recovery=True, dcs_last_seen=0,
                     timeline=2, wal_position=10, nofailover=False,
-                    watchdog_failed=False):
+                    watchdog_failed=False, failover_priority=1):
     def fetch_node_status(e):
         tags = {}
         if nofailover:
             tags['nofailover'] = True
+        tags['failover_priority'] = failover_priority
         return _MemberStatus(e, reachable, in_recovery, wal_position,
                              {'tags': tags, 'watchdog_failed': watchdog_failed,
                               'dcs_last_seen': dcs_last_seen, 'timeline': timeline})
@@ -153,6 +154,7 @@ zookeeper:
                                   'postmaster_start_time': str(postmaster_start_time)}
         self.watchdog = Watchdog(self.config)
         self.request = lambda *args, **kwargs: requests_get(args[0].api_url, *args[1:], **kwargs)
+        self.failover_priority = 1
 
 
 def run_async(self, func, args=()):
@@ -1036,6 +1038,11 @@ class TestHa(PostgresInit):
         self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.fetch_node_status = get_node_status(in_recovery=False)  # accessible, not in_recovery
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
+        self.ha.fetch_node_status = get_node_status(failover_priority=2)  # accessible, in_recovery, higher priority
+        self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
+        # if there is a higher-priority node but it has a lower WAL position then this node should race
+        self.ha.fetch_node_status = get_node_status(failover_priority=6, wal_position=9)
+        self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.fetch_node_status = get_node_status(wal_position=11)  # accessible, in_recovery, wal position ahead
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         # in synchronous_mode consider itself healthy if the former leader is accessible in read-only and ahead of us
@@ -1051,7 +1058,9 @@ class TestHa(PostgresInit):
             self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.patroni.nofailover = True
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
-        self.ha.patroni.nofailover = False
+        self.ha.patroni.nofailover = None
+        self.ha.patroni.failover_priority = 0
+        self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
 
     def test_fetch_node_status(self):
         member = Member(0, 'test', 1, {'api_url': 'http://127.0.0.1:8011/patroni'})

tests/test_patroni.py

@@ -179,10 +179,42 @@ class TestPatroni(unittest.TestCase):
         self.assertTrue(self.p.noloadbalance)
 
     def test_nofailover(self):
-        self.p.tags['nofailover'] = True
-        self.assertTrue(self.p.nofailover)
-        self.p.tags['nofailover'] = None
-        self.assertFalse(self.p.nofailover)
+        for (nofailover, failover_priority, expected) in [
+            # Without any tags, default is False
+            (None, None, False),
+            # Setting `nofailover: True` has precedence
+            (True, 0, True),
+            (True, 1, True),
+            # Similarly, setting `nofailover: False` has precedence
+            (False, 0, False),
+            (False, 1, False),
+            # Only when we have `nofailover: None` should we got based on priority
+            (None, 0, True),
+            (None, 1, False),
+        ]:
+            with self.subTest(nofailover=nofailover, failover_priority=failover_priority, expected=expected):
+                self.p.tags['nofailover'] = nofailover
+                self.p.tags['failover_priority'] = failover_priority
+                self.assertEqual(self.p.nofailover, expected)
+
+    def test_failover_priority(self):
+        for (nofailover, failover_priority, expected) in [
+            # Without any tags, default is 1
+            (None, None, 1),
+            # Setting `nofailover: True` has precedence (value 0)
+            (True, 0, 0),
+            (True, 1, 0),
+            # Setting `nofailover: False` and `failover_priority: None` gives 1
+            (False, None, 1),
+            # Normal function of failover_priority
+            (None, 0, 0),
+            (None, 1, 1),
+            (None, 2, 2),
+        ]:
+            with self.subTest(nofailover=nofailover, failover_priority=failover_priority, expected=expected):
+                self.p.tags['nofailover'] = nofailover
+                self.p.tags['failover_priority'] = failover_priority
+                self.assertEqual(self.p.failover_priority, expected)
 
     def test_replicatefrom(self):
         self.assertIsNone(self.p.replicatefrom)

tests/test_validator.py

@@ -325,3 +325,35 @@ class TestValidator(unittest.TestCase):
         output = "\n".join(errors)
         self.assertEqual(['postgresql.bin_dir', 'postgresql.bin_name.postgres', 'raft.bind_addr', 'raft.self_addr'],
                          parse_output(output))
+
+    def test_one_of(self, _, __):
+        c = copy.deepcopy(config)
+        # Providing neither is fine
+        del c["tags"]["nofailover"]
+        errors = schema(c)
+        self.assertNotIn("tags  Multiple of ('nofailover', 'failover_priority') provided", errors)
+        # Just nofailover is fine
+        c["tags"]["nofailover"] = False
+        errors = schema(c)
+        self.assertNotIn("tags  Multiple of ('nofailover', 'failover_priority') provided", errors)
+        # Just failover_priority is fine
+        del c["tags"]["nofailover"]
+        c["tags"]["failover_priority"] = 1
+        errors = schema(c)
+        self.assertNotIn("tags  Multiple of ('nofailover', 'failover_priority') provided", errors)
+        # Providing both is not fine
+        c["tags"]["nofailover"] = False
+        errors = schema(c)
+        self.assertIn("tags  Multiple of ('nofailover', 'failover_priority') provided", errors)
+
+    def test_failover_priority_int(self, *args):
+        c = copy.deepcopy(config)
+        del c["tags"]["nofailover"]
+        c["tags"]["failover_priority"] = 'a string'
+        errors = schema(c)
+        self.assertIn('tags.failover_priority a string is not an integer', errors)
+        c = copy.deepcopy(config)
+        del c["tags"]["nofailover"]
+        c["tags"]["failover_priority"] = -6
+        errors = schema(c)
+        self.assertIn('tags.failover_priority -6 didn\'t pass validation: Wrong value', errors)