From db9b5962ec5b921c4c98823f623084f6be58b648 Mon Sep 17 00:00:00 2001 From: Ants Aasma Date: Thu, 29 Sep 2022 14:39:51 +0300 Subject: [PATCH] Avoid cloning while bootstrap is running (#2419) If cluster has a create replica method that does not require a leader it can get triggered while bootstrap is running. If that method comes up with an accessible cluster faster than the bootstrap completes it will get promoted as the leader, only to lose leader lock when bootstrap completes. To fix this we only consider leaderless create replica methods if sysid is non-empty, i.e. there is no bootstrap running. --- patroni/ha.py | 4 +++- tests/test_ha.py | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/patroni/ha.py b/patroni/ha.py index 507d38ca..3ec672b9 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -273,7 +273,9 @@ class Ha(object): else: create_replica_methods = self.get_standby_cluster_config().get('create_replica_methods', []) \ if self.is_standby_cluster() else None - if self.state_handler.can_create_replica_without_replication_connection(create_replica_methods): + can_bootstrap = self.state_handler.can_create_replica_without_replication_connection(create_replica_methods) + concurrent_bootstrap = self.cluster.initialize == "" + if can_bootstrap and not concurrent_bootstrap: msg = 'bootstrap (without leader)' return self._async_executor.try_run_async(msg, self.clone) or 'trying to ' + msg return 'waiting for {0}leader to bootstrap'.format('standby_' if self.is_standby_cluster() else '') diff --git a/tests/test_ha.py b/tests/test_ha.py index 0203d9ca..b5c496c1 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -45,6 +45,10 @@ def get_cluster_not_initialized_without_leader(cluster_config=None): return get_cluster(None, None, [], None, SyncState(None, None, None), cluster_config) +def get_cluster_bootstrapping_without_leader(cluster_config=None): + return get_cluster("", None, [], None, SyncState(None, None, None), cluster_config) + + def get_cluster_initialized_without_leader(leader=False, failover=None, sync=None, cluster_config=None): m1 = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5435/postgres', 'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location': 4}) @@ -469,6 +473,11 @@ class TestHa(PostgresInit): self.p.can_create_replica_without_replication_connection = MagicMock(return_value=True) self.assertEqual(self.ha.bootstrap(), 'trying to bootstrap (without leader)') + def test_bootstrap_not_running_concurrently(self): + self.ha.cluster = get_cluster_bootstrapping_without_leader() + self.p.can_create_replica_without_replication_connection = MagicMock(return_value=True) + self.assertEqual(self.ha.bootstrap(), 'waiting for leader to bootstrap') + def test_bootstrap_initialize_lock_failed(self): self.ha.cluster = get_cluster_not_initialized_without_leader() self.assertEqual(self.ha.bootstrap(), 'failed to acquire initialize lock')