Delay _process_quorum_replication by loop_wait seconds after promote

It takes some time for existing standbys to start streaming from the
new primary and we want to do our best to not empty the /sync key before
that.

patroni/ha.py

@@ -147,6 +147,7 @@ class Ha(object):
         self._is_leader_lock = RLock()
         self._failsafe = Failsafe(patroni.dcs)
         self._was_paused = False
+        self._promote_time = 0
         self._leader_timeline = None
         self.recovering = False
         self._async_response = CriticalTask()
@@ -196,6 +197,8 @@ class Ha(object):
     def set_is_leader(self, value: bool) -> None:
         with self._is_leader_lock:
             self._is_leader = time.time() + self.dcs.ttl if value else 0
+            if not value:
+                self._promote_time = 0
 
     def load_cluster_from_dcs(self) -> None:
         cluster = self.dcs.get_cluster()
@@ -702,7 +705,16 @@ class Ha(object):
     def process_sync_replication(self) -> None:
         """Process synchronous replication beahvior on the primary."""
         if self.is_quorum_commit_mode():
-            self._process_quorum_replication()
+            # The synchronous_standby_names was adjusted right before promote.
+            # After that, when postgres has become a primary, we need to reflect this change
+            # in the /sync key. Further changes of synchronous_standby_names and /sync key should
+            # be postponed for `loop_wait` seconds, to give a chance to some replicas to start streaming.
+            # In opposite case the /sync key will end up without synchronous nodes.
+            if self.state_handler.is_leader():
+                if self._promote_time == 0 or time.time() - self._promote_time > self.dcs.loop_wait:
+                    self._process_quorum_replication()
+                if self._promote_time == 0:
+                    self._promote_time = time.time()
         elif self.is_synchronous_mode():
             self._process_multisync_replication()
         else:

tests/test_ha.py

@@ -1500,6 +1500,7 @@ class TestHa(PostgresInit):
         self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': None})
         self.assertEqual(mock_set_sync.call_count, 0)
 
+        self.ha._promote_time = 1
         mock_write_sync = self.ha.dcs.write_sync_state = Mock(side_effect=[SyncState.empty(), None])
         # Test /sync key is attempted to set and succeed when missing or invalid
         with patch.object(SyncState, 'is_empty', Mock(side_effect=[True, False])):