From 278bf9852b0d74bd2a2f166bce6f7a007cee9343 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 5 Aug 2019 15:08:04 +0200 Subject: [PATCH] Release 1.6.0 (#1131) * Implement missing tests and do a few minor fixes * Bump version to 1.6.0 * Update release notes --- docs/ENVIRONMENT.rst | 2 + docs/README.rst | 2 + docs/SETTINGS.rst | 2 +- docs/releases.rst | 112 +++++++++++++++++++++++++++++++++++ patroni/ctl.py | 11 ++-- patroni/dcs/__init__.py | 3 +- patroni/dcs/consul.py | 12 ++-- patroni/postgresql/config.py | 13 ++-- patroni/version.py | 2 +- tests/test_consul.py | 3 + tests/test_ctl.py | 12 ++-- 11 files changed, 149 insertions(+), 25 deletions(-) diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst index 52cdd759..b08b3dbe 100644 --- a/docs/ENVIRONMENT.rst +++ b/docs/ENVIRONMENT.rst @@ -91,6 +91,8 @@ PostgreSQL - **PATRONI\_REPLICATION\_PASSWORD**: replication password; the user will be created during initialization. - **PATRONI\_SUPERUSER\_USERNAME**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres. Also this user is used by pg_rewind. - **PATRONI\_SUPERUSER\_PASSWORD**: password for the superuser, set during initialization (initdb). +- **PATRONI\_REWIND\_USERNAME**: name for the user for ``pg_rewind``; the user will be created during initialization of postgres 11+ and all necessary `permissions `__ will be granted. +- **PATRONI\_REWIND\_PASSWORD**: password for the user for ``pg_rewind``; the user will be created during initialization. REST API -------- diff --git a/docs/README.rst b/docs/README.rst index 8e0e3a7e..6a86d8f2 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -33,6 +33,8 @@ To install requirements on a Mac, run the following: brew install postgresql etcd haproxy libyaml python +.. _psycopg2_install_options: + **Psycopg2** Starting from `psycopg2-2.8 `__ the binary version of psycopg2 will no longer be installed by default. Installing it from the source code requires C compiler and postgres+python dev packages. diff --git a/docs/SETTINGS.rst b/docs/SETTINGS.rst index ea3bdf54..4038a95d 100644 --- a/docs/SETTINGS.rst +++ b/docs/SETTINGS.rst @@ -140,7 +140,7 @@ PostgreSQL - **username**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication - **password**: replication password; the user will be created during initialization. - **rewind**: - - **username**: name for the user for ``pg_rewind``; the user will be created during initialization of postgres 11+ and all necessary `permissions `__ will be granted. + - **username**: name for the user for ``pg_rewind``; the user will be created during initialization of postgres 11+ and all necessary `permissions `__ will be granted. - **password**: password for the user for ``pg_rewind``; the user will be created during initialization. - **callbacks**: callback scripts to run on certain actions. Patroni will pass the action, role and cluster name. (See scripts/aws.py as an example of how to write them.) - **on\_reload**: run this script when configuration reload is triggered. diff --git a/docs/releases.rst b/docs/releases.rst index 17b99283..26a00f59 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -3,6 +3,118 @@ Release notes ============= +Version 1.6.0 +------------- + +This version adds compatibility with PostgreSQL 12, makes is possible to run pg_rewind without superuser on PostgreSQL 11 and newer, and enables IPv6 support. + + +**New features** + +- Psycopg2 was removed from requirements and must be installed independently (Alexander Kukushkin) + + Starting from 2.8.0 ``psycopg2`` was split into two different packages, ``psycopg2``, and ``psycopg2-binary``, which could be installed at the same time into the same place on the filesystem. In order to decrease dependency hell problem, we let a user choose how to install it. There are a few options available, please consult the :ref:`documentation `. + +- Compatibility with PostgreSQL 12 (Alexander Kukushkin) + + Starting from PostgreSQL 12 there is no ``recovery.conf`` anymore and all former recovery parameters are converted into `GUC `_. In order to protect from ``ALTER SYSTEM SET primary_conninfo`` or similar, Patroni will parse ``postgresql.auto.conf`` and remove all standby and recovery parameters from there. Patroni config remains backward compatible. For example despite ``restore_command`` being a GUC, one can still specify it in the ``postgresql.recovery_conf.restore_command`` section and Patroni will write it into ``postgresql.conf`` for PostgreSQL 12. + +- Make it possible to use ``pg_rewind`` without superuser on PostgreSQL 11 and newer (Alexander Kukushkin) + + If you want to use this feature please define ``username`` and ``password`` in the ``postgresql.authentication.rewind`` section of Patroni configuration file. For an already existing cluster you will have to create the user manually and ``GRANT EXECUTE`` permission on a few functions. You can find more details in the PostgreSQL `documentation `__. + +- Do a smart comparison of actual and desired ``primary_conninfo`` values on replicas (Alexander Kukushkin) + + It might help to avoid replica restart when you are converting an already existing primary-standby cluster to one managed by Patroni + +- IPv6 support (Alexander Kukushkin) + + There were two major issues. Patroni REST API service was listening only on ``0.0.0.0`` and IPv6 IP addresses used in the ``api_url`` and ``conn_url`` were not properly quoted. + +- Kerberos support (Ajith Vilas, Alexander Kukushkin) + + It makes possible using Kerberos authentication between Postgres nodes instead of defining passwords in Patroni configuration file + +- Manage ``pg_ident.conf`` (Alexander Kukushkin) + + This functionality works similarly to ``pg_hba.conf``: if the ``postgresql.pg_ident`` is defined in the config file or DCS, Patroni will write its value to ``pg_ident.conf``, however, if ``postgresql.parameters.ident_file`` is defined, Patroni will assume that ``pg_ident`` is managed from outside and not update the file. + + + **Improvements in REST API** + +- Added ``/health`` endpoint (Wilfried Roset) + + It will return an HTTP status code only if PostgreSQL is running + +- Added ``/read-only`` and ``/read-write`` endpoints (Julien Riou) + + The ``/read-only`` endpoint enables reads balanced across replicas and the primary. The ``/read-write`` endpoint is an alias for ``/primary``, ``/leader`` and ``/master``. + +- Use ``SSLContext`` to wrap the REST API socket (Julien Riou) + + Usage of ``ssl.wrap_socket()`` is deprecated and was still allowing soon-to-be-deprecated protocols like TLS 1.1. + + +**Logging improvements** + +- Two-step logging (Alexander Kukushkin) + + All log messages are first written into the in-memory queue and later they are asynchronously flushed into the stderr or file from a separate thread. The maximum queue size is limited (configurable). If the limit is reached, Patroni will start losing logs, which is still better than blocking the HA loop. + +- Enable debug logging for GET/OPTIONS API calls together with latency (Jan Tomsa) + + It will help with debugging of health-checks performed by HAProxy, Consul or other tooling that decides which node is the primary/replica. + +- Log exceptions caught in Retry (Daniel Kucera) + + Log the final exception when either the number of attempts or the timeout were reached. It will hopefully help to debug some issues when communication to DCS fails. + + +**Improvements in patronictl** + +- Enhance dialogues for scheduled switchover and restart (Rafia Sabih) + + Previously dialogues did not take into account scheduled actions and therefore were misleading. + +- Check if config file exists (Wilfried Roset) + + Be verbose about configuration file when the given filename does not exists, instead of ignoring silently (which can lead to misunderstanding). + +- Add fallback value for ``EDITOR`` (Wilfried Roset) + + When the ``EDITOR`` environment variable was not defined, ``patronictl edit-config`` was failing with `PatroniCtlException`. The new strategy is to try ``editor`` and than ``vi``, which should be available on most systems. + + +**Improvements in Consul support** + +- Allow to specify Consul consistency mode (Jan Tomsa) + + You can read more about consistency mode `here `__. + +- Reload Consul config on SIGHUP (Cameron Daniel, Alexander Kukushkin) + + It is especially useful when somebody is changing the value of ``token``. + + +**Bugfixes** + +- Fix corner case in switchover/failover (Sharoon Thomas) + + The variable ``scheduled_at`` may be undefined if REST API is not accessible and we are using DCS as a fallback. + +- Open trust to localhost in ``pg_hba.conf`` during custom bootstrap (Alexander Kukushkin) + + Previously it was open only to unix_socket, which was causing a lot of errors: ``FATAL: no pg_hba.conf entry for replication connection from host "127.0.0.1", user "replicator"`` + +- Consider synchronous node as healthy even when the former leader is ahead (Alexander Kukushkin) + + If the primary loses access to the DCS, it restarts Postgres in read-only, but it might happen that other nodes can still access the old primary via the REST API. Such a situation was causing the synchronous standby not to promote because the old primary was reporting WAL position ahead of the synchronous standby. + +- Standby cluster bugfixes (Alexander Kukushkin) + + Make it possible to bootstrap a replica in a standby cluster when the standby_leader is not accessible and a few other minor fixes. + + Version 1.5.6 ------------- diff --git a/patroni/ctl.py b/patroni/ctl.py index 6b899f3d..a4df213a 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -116,7 +116,10 @@ option_insecure = click.option('-k', '--insecure', is_flag=True, help='Allow con @option_insecure @click.pass_context def ctl(ctx, config_file, dcs, insecure): - logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=os.environ.get('LOGLEVEL', 'WARNING')) + level = 'WARNING' + for name in ('LOGLEVEL', 'PATRONI_LOGLEVEL', 'PATRONI_LOG_LEVEL'): + level = os.environ.get(name, level) + logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=level) logging.captureWarnings(True) # Capture eventual SSL warning ctx.obj = load_config(config_file, dcs) # backward compatibility for configuration file where ctl section is not define @@ -1109,9 +1112,9 @@ def invoke_editor(before_editing, cluster_name): :param before_editing: human representation before editing :returns tuple of human readable and parsed datastructure after changes """ - if 'EDITOR' in os.environ: - editor_cmd = os.environ.get('EDITOR') - else: + + editor_cmd = os.environ.get('EDITOR') + if not editor_cmd: for editor in ('editor', 'vi'): editor_cmd = find_executable(editor) if editor_cmd: diff --git a/patroni/dcs/__init__.py b/patroni/dcs/__init__.py index 624e29d4..3fa7239a 100644 --- a/patroni/dcs/__init__.py +++ b/patroni/dcs/__init__.py @@ -244,7 +244,8 @@ class Leader(namedtuple('Leader', 'index,session,member')): version = self.member.data.get('version') if version: try: - if tuple(map(int, version.split('.'))) >= (1, 5, 6): + # 1.5.6 is the last version which doesn't expose checkpoint_after_promote: false + if tuple(map(int, version.split('.'))) > (1, 5, 6): return self.member.data['role'] == 'master' and 'checkpoint_after_promote' not in self.member.data except Exception: logger.debug('Failed to parse Patroni version %s', version) diff --git a/patroni/dcs/consul.py b/patroni/dcs/consul.py index 8147ec55..cb509ac7 100644 --- a/patroni/dcs/consul.py +++ b/patroni/dcs/consul.py @@ -135,9 +135,10 @@ class ConsulClient(base.Consul): kwargs['token'] = self.token return HTTPClient(**kwargs) - def set_token(self, token): - self.token = token - self.http.token = self.token + def reload_config(self, config): + self.http.token = self.token = config.get('token') + self.consistency = config.get('consistency', 'default') + self.dc = config.get('dc') def catch_consul_errors(func): @@ -237,11 +238,8 @@ class Consul(AbstractDCS): time.sleep(5) def reload_config(self, config): - consul = config.get('consul') - if consul: - self._client.set_token(consul.get('token')) - super(Consul, self).reload_config(config) + self._client.reload_config(config.get('consul', {})) def set_ttl(self, ttl): if self._client.http.set_ttl(ttl/2.0): # Consul multiplies the TTL by 2x diff --git a/patroni/postgresql/config.py b/patroni/postgresql/config.py index ccc5b679..ff10e2f2 100644 --- a/patroni/postgresql/config.py +++ b/patroni/postgresql/config.py @@ -135,12 +135,13 @@ def parse_dsn(value): ret = conninfo_parse(value) if ret: - requiressl = ret.pop('requiressl', None) - if requiressl == '1': - ret['sslmode'] = 'require' - elif requiressl is not None: - ret['sslmode'] = 'prefer' - ret.setdefault('sslmode', 'prefer') + if 'sslmode' not in ret: # allow sslmode to take precedence over requiressl + requiressl = ret.pop('requiressl', None) + if requiressl == '1': + ret['sslmode'] = 'require' + elif requiressl is not None: + ret['sslmode'] = 'prefer' + ret.setdefault('sslmode', 'prefer') if 'dbname' in ret: del ret['dbname'] return ret diff --git a/patroni/version.py b/patroni/version.py index 43fa9aa9..bcd8d54e 100644 --- a/patroni/version.py +++ b/patroni/version.py @@ -1 +1 @@ -__version__ = '1.5.6' +__version__ = '1.6.0' diff --git a/tests/test_consul.py b/tests/test_consul.py index c6571214..785e80f3 100644 --- a/tests/test_consul.py +++ b/tests/test_consul.py @@ -197,3 +197,6 @@ class TestConsul(unittest.TestCase): d['state'] = 'running' d['role'] = 'bla' self.assertIsNone(self.c.update_service({}, d)) + + def test_reload_config(self): + self.c.reload_config({'consul': {'token': 'foo'}, 'loop_wait': 10, 'ttl': 30, 'retry_timeout': 10}) diff --git a/tests/test_ctl.py b/tests/test_ctl.py index db04cbf6..041488ae 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -25,9 +25,9 @@ CONFIG_FILE_PATH = './test-ctl.yaml' def test_rw_config(): runner = CliRunner() with runner.isolated_filesystem(): - store_config({'etcd': {'host': 'localhost:2379'}}, CONFIG_FILE_PATH + '/dummy') sys.argv = ['patronictl.py', ''] load_config(CONFIG_FILE_PATH + '/dummy', None) + store_config({'etcd': {'host': 'localhost:2379'}}, CONFIG_FILE_PATH + '/dummy') load_config(CONFIG_FILE_PATH + '/dummy', '0.0.0.0') os.remove(CONFIG_FILE_PATH + '/dummy') os.rmdir(CONFIG_FILE_PATH) @@ -95,7 +95,8 @@ class TestCtl(unittest.TestCase): assert result.exit_code == 1 # Aborting scheduled switchover, as we answer NO to the confirmation - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--scheduled', '2015-01-01T12:00:00+01:00'], input='leader\nother\n\nN') + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--scheduled', '2015-01-01T12:00:00+01:00'], + input='leader\nother\n\nN') assert result.exit_code == 1 # Target and source are equal @@ -544,9 +545,10 @@ class TestCtl(unittest.TestCase): @patch('subprocess.call', return_value=1) def test_invoke_editor(self, mock_subprocess_call): - for e in ('', 'false'): - os.environ['EDITOR'] = e - self.assertRaises(PatroniCtlException, invoke_editor, 'foo: bar\n', 'test') + os.environ.pop('EDITOR', None) + for e in ('', '/bin/vi'): + with patch('patroni.ctl.find_executable', Mock(return_value=e)): + self.assertRaises(PatroniCtlException, invoke_editor, 'foo: bar\n', 'test') @patch('patroni.ctl.get_dcs') def test_show_config(self, mock_get_dcs):