From 7bf60b64b094ef3f03678b835afd703ad66f0738 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 17 Aug 2020 10:45:02 +0200 Subject: [PATCH] Compatibility with PostgreSQL 13 (#1654) So far Patroni was enforcing the same value of `wal_keep_segments` on all nodes in the cluster. If the parameter was missing from the global configuration it was using the default value `8`. In pg13 beta3 the `wal_keep_segments` was renamed to the `wal_keep_size` and it broke Patroni. If `wal_keep_segments` happened to be present in the configuration for pg13, Paroni will recalculate the value to `wal_keep_size` assuming that the `wal_segment_size` is 16MB. Sure, it is possible to get the real value of `wal_segment_size` from pg_control, but since we are dealing with the case of misconfiguration it is not worse time spend on it. --- docs/dynamic_configuration.rst | 1 + docs/rest_api.rst | 6 +----- patroni/config.py | 3 ++- patroni/postgresql/__init__.py | 2 +- patroni/postgresql/config.py | 17 +++++++++++++++-- 5 files changed, 20 insertions(+), 9 deletions(-) diff --git a/docs/dynamic_configuration.rst b/docs/dynamic_configuration.rst index 6f350b18..4a0e2136 100644 --- a/docs/dynamic_configuration.rst +++ b/docs/dynamic_configuration.rst @@ -35,6 +35,7 @@ For the parameters below, PostgreSQL does not require equal values among the mas - max_wal_senders: 5 - max_replication_slots: 5 - wal_keep_segments: 8 +- wal_keep_size: 128MB These parameters are validated to ensure they are sane, or meet a minimum value. diff --git a/docs/rest_api.rst b/docs/rest_api.rst index 25f1a7df..74003540 100644 --- a/docs/rest_api.rst +++ b/docs/rest_api.rst @@ -193,7 +193,6 @@ Config endpoint "parameters": { "hot_standby": "on", "wal_log_hints": "on", - "wal_keep_segments": 8, "wal_level": "hot_standby", "max_wal_senders": 5, "max_replication_slots": 5, @@ -221,7 +220,6 @@ Config endpoint "parameters": { "hot_standby": "on", "wal_log_hints": "on", - "wal_keep_segments": 8, "wal_level": "hot_standby", "max_wal_senders": 5, "max_replication_slots": 5, @@ -273,7 +271,6 @@ If you want to remove (reset) some setting just patch it with ``null``: "parameters": { "hot_standby": "on", "unix_socket_directories": ".", - "wal_keep_segments": 8, "wal_level": "hot_standby", "wal_log_hints": "on", "max_wal_senders": 5, @@ -289,7 +286,7 @@ The above call removes ``postgresql.parameters.max_connections`` from the dynami .. code-block:: bash $ curl -s -XPUT -d \ - '{"maximum_lag_on_failover":1048576,"retry_timeout":10,"postgresql":{"use_slots":true,"use_pg_rewind":true,"parameters":{"hot_standby":"on","wal_log_hints":"on","wal_keep_segments":8,"wal_level":"hot_standby","unix_socket_directories":".","max_wal_senders":5}},"loop_wait":3,"ttl":20}' \ + '{"maximum_lag_on_failover":1048576,"retry_timeout":10,"postgresql":{"use_slots":true,"use_pg_rewind":true,"parameters":{"hot_standby":"on","wal_log_hints":"on","wal_level":"hot_standby","unix_socket_directories":".","max_wal_senders":5}},"loop_wait":3,"ttl":20}' \ http://localhost:8008/config | jq . { "ttl": 20, @@ -300,7 +297,6 @@ The above call removes ``postgresql.parameters.max_connections`` from the dynami "parameters": { "hot_standby": "on", "unix_socket_directories": ".", - "wal_keep_segments": 8, "wal_level": "hot_standby", "wal_log_hints": "on", "max_wal_senders": 5 diff --git a/patroni/config.py b/patroni/config.py index 10a81f3c..8253d18d 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -77,7 +77,8 @@ class Config(object): 'postgresql': { 'bin_dir': '', 'use_slots': True, - 'parameters': CaseInsensitiveDict({p: v[0] for p, v in ConfigHandler.CMDLINE_OPTIONS.items()}) + 'parameters': CaseInsensitiveDict({p: v[0] for p, v in ConfigHandler.CMDLINE_OPTIONS.items() + if p not in ('wal_keep_segments', 'wal_keep_size')}) }, 'watchdog': { 'mode': 'automatic', diff --git a/patroni/postgresql/__init__.py b/patroni/postgresql/__init__.py index acaae46c..31361793 100644 --- a/patroni/postgresql/__init__.py +++ b/patroni/postgresql/__init__.py @@ -470,7 +470,7 @@ class Postgresql(object): self.config.replace_pg_ident() options = ['--{0}={1}'.format(p, configuration[p]) for p in self.config.CMDLINE_OPTIONS - if p in configuration and p != 'wal_keep_segments'] + if p in configuration and p not in ('wal_keep_segments', 'wal_keep_size')] if self.cancellable.is_cancelled: return False diff --git a/patroni/postgresql/config.py b/patroni/postgresql/config.py index 46a754df..cbeb3c2e 100644 --- a/patroni/postgresql/config.py +++ b/patroni/postgresql/config.py @@ -294,6 +294,7 @@ class ConfigHandler(object): 'max_connections': (100, lambda v: int(v) >= 25, 90100), 'max_wal_senders': (10, lambda v: int(v) >= 3, 90100), 'wal_keep_segments': (8, lambda v: int(v) >= 1, 90100), + 'wal_keep_size': ('128MB', lambda v: parse_int(v, 'MB') >= 16, 130000), 'max_prepared_transactions': (0, lambda v: int(v) >= 0, 90100), 'max_locks_per_transaction': (64, lambda v: int(v) >= 32, 90100), 'track_commit_timestamp': ('off', lambda v: parse_bool(v) is not None, 90500), @@ -830,8 +831,20 @@ class ConfigHandler(object): parameters.pop('synchronous_standby_names', None) else: parameters['synchronous_standby_names'] = self._synchronous_standby_names - if self._postgresql.major_version >= 90600 and parameters['wal_level'] == 'hot_standby': - parameters['wal_level'] = 'replica' + + # Handle hot_standby <-> replica rename + if parameters.get('wal_level') == ('hot_standby' if self._postgresql.major_version >= 90600 else 'replica'): + parameters['wal_level'] = 'replica' if self._postgresql.major_version >= 90600 else 'hot_standby' + + # Try to recalcualte wal_keep_segments <-> wal_keep_size assuming that typical wal_segment_size is 16MB. + # The real segment size could be estimated from pg_control, but we don't really care, because the only goal of + # this exercise is improving cross version compatibility and user must set the correct parameter in the config. + if self._postgresql.major_version >= 130000: + wal_keep_segments = parameters.pop('wal_keep_segments', self.CMDLINE_OPTIONS['wal_keep_segments'][0]) + parameters.setdefault('wal_keep_size', str(wal_keep_segments * 16) + 'MB') + else: + wal_keep_size = parse_int(parameters.pop('wal_keep_size', self.CMDLINE_OPTIONS['wal_keep_size'][0]), 'MB') + parameters.setdefault('wal_keep_segments', int((wal_keep_size + 8) / 16)) ret = CaseInsensitiveDict({k: v for k, v in parameters.items() if not self._postgresql.major_version or self._postgresql.major_version >= self.CMDLINE_OPTIONS.get(k, (0, 1, 90100))[2]}) ret.update({k: os.path.join(self._config_dir, ret[k]) for k in ('hba_file', 'ident_file') if k in ret})