From df18885f2090219bdebde9829ea5dfb13d401a56 Mon Sep 17 00:00:00 2001 From: Israel Date: Wed, 31 May 2023 08:54:54 -0300 Subject: [PATCH 01/22] Extend Postgres GUCs validator (#2671) * Use YAML files to validate Postgres GUCs through Patroni. Patroni used to have a static list of Postgres GUCs validators in `patroni.postgresql.validator`. One problem with that approach, for example, is that it would not allow GUCs from custom Postgres builds to be validated/accepted. The idea that we had to work around that issue was to move the validators from the source code to an external and extendable source. With that Patroni will start reading the current validators from that external source plus whatever custom validators are found. From this commit onwards Patroni will read and parse all YAML files that are found under the `patroni/postgresql/available_parameters` directory to build its Postgres GUCs validation rules. All the details about how this work can be found in the docstring of the introduced function `_load_postgres_gucs_validators`. --- patroni.spec | 5 +- patroni/postgresql/__init__.py | 16 + .../available_parameters/0_postgres.yml | 1710 +++++++++++++++++ patroni/postgresql/config.py | 6 +- patroni/postgresql/validator.py | 813 ++++---- setup.py | 5 +- tests/__init__.py | 11 +- tests/test_bootstrap.py | 3 +- tests/test_patroni.py | 2 + tests/test_postgresql.py | 218 ++- tests/test_sync.py | 4 +- 11 files changed, 2341 insertions(+), 452 deletions(-) create mode 100644 patroni/postgresql/available_parameters/0_postgres.yml diff --git a/patroni.spec b/patroni.spec index 121e6f5d..b5c414a2 100644 --- a/patroni.spec +++ b/patroni.spec @@ -16,7 +16,10 @@ def hiddenimports(): a = Analysis(['patroni/__main__.py'], pathex=[], binaries=None, - datas=None, + datas=[ + ('patroni/postgresql/available_parameters/*.yml', 'patroni/postgresql/available_parameters'), + ('patroni/postgresql/available_parameters/*.yaml', 'patroni/postgresql/available_parameters'), + ], hiddenimports=hiddenimports(), hookspath=[], runtime_hooks=[], diff --git a/patroni/postgresql/__init__.py b/patroni/postgresql/__init__.py index 6d2d18af..b2423c89 100644 --- a/patroni/postgresql/__init__.py +++ b/patroni/postgresql/__init__.py @@ -26,6 +26,7 @@ from .slots import SlotsHandler from .sync import SyncHandler from .. import psycopg from ..async_executor import CriticalTask +from ..collections import CaseInsensitiveSet from ..dcs import Cluster, Leader, Member from ..exceptions import PostgresConnectionException from ..utils import Retry, RetryFailedError, polling_loop, data_directory_is_empty, parse_int @@ -211,6 +212,11 @@ class Postgresql(object): return ("SELECT " + self.TL_LSN + ", {2}").format(self.wal_name, self.lsn_name, extra) + @property + def available_gucs(self) -> CaseInsensitiveSet: + """GUCs available in this Postgres server.""" + return self._get_gucs() + def _version_file_exists(self) -> bool: return not self.data_directory_empty() and os.path.isfile(self._version_file) @@ -1265,3 +1271,13 @@ class Postgresql(object): self.slots_handler.schedule() self.citus_handler.schedule_cache_rebuild() self._sysid = '' + + def _get_gucs(self) -> CaseInsensitiveSet: + """Get all available GUCs based on ``postgres --describe-config`` output. + + :returns: all available GUCs in the local Postgres server. + """ + cmd = [self.pgcommand('postgres'), '--describe-config'] + return CaseInsensitiveSet({ + line.split('\t')[0] for line in subprocess.check_output(cmd).decode('utf-8').strip().split('\n') + }) diff --git a/patroni/postgresql/available_parameters/0_postgres.yml b/patroni/postgresql/available_parameters/0_postgres.yml new file mode 100644 index 00000000..5afd2efb --- /dev/null +++ b/patroni/postgresql/available_parameters/0_postgres.yml @@ -0,0 +1,1710 @@ +parameters: + allow_in_place_tablespaces: + - type: Bool + version_from: 150000 + allow_system_table_mods: + - type: Bool + version_from: 90300 + application_name: + - type: String + version_from: 90300 + archive_command: + - type: String + version_from: 90300 + archive_library: + - type: String + version_from: 150000 + archive_mode: + - type: Bool + version_from: 90300 + version_till: 90500 + - type: EnumBool + version_from: 90500 + possible_values: + - always + archive_timeout: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 1073741823 + unit: s + array_nulls: + - type: Bool + version_from: 90300 + authentication_timeout: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 600 + unit: s + autovacuum: + - type: Bool + version_from: 90300 + autovacuum_analyze_scale_factor: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 100 + autovacuum_analyze_threshold: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + autovacuum_freeze_max_age: + - type: Integer + version_from: 90300 + min_val: 100000 + max_val: 2000000000 + autovacuum_max_workers: + - type: Integer + version_from: 90300 + version_till: 90600 + min_val: 1 + max_val: 8388607 + - type: Integer + version_from: 90600 + min_val: 1 + max_val: 262143 + autovacuum_multixact_freeze_max_age: + - type: Integer + version_from: 90300 + min_val: 10000 + max_val: 2000000000 + autovacuum_naptime: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 2147483 + unit: s + autovacuum_vacuum_cost_delay: + - type: Integer + version_from: 90300 + version_till: 120000 + min_val: -1 + max_val: 100 + unit: ms + - type: Real + version_from: 120000 + min_val: -1 + max_val: 100 + unit: ms + autovacuum_vacuum_cost_limit: + - type: Integer + version_from: 90300 + min_val: -1 + max_val: 10000 + autovacuum_vacuum_insert_scale_factor: + - type: Real + version_from: 130000 + min_val: 0 + max_val: 100 + autovacuum_vacuum_insert_threshold: + - type: Integer + version_from: 130000 + min_val: -1 + max_val: 2147483647 + autovacuum_vacuum_scale_factor: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 100 + autovacuum_vacuum_threshold: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + autovacuum_work_mem: + - type: Integer + version_from: 90400 + min_val: -1 + max_val: 2147483647 + unit: kB + backend_flush_after: + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 256 + unit: 8kB + backslash_quote: + - type: EnumBool + version_from: 90300 + possible_values: + - safe_encoding + backtrace_functions: + - type: String + version_from: 130000 + bgwriter_delay: + - type: Integer + version_from: 90300 + min_val: 10 + max_val: 10000 + unit: ms + bgwriter_flush_after: + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 256 + unit: 8kB + bgwriter_lru_maxpages: + - type: Integer + version_from: 90300 + version_till: 100000 + min_val: 0 + max_val: 1000 + - type: Integer + version_from: 100000 + min_val: 0 + max_val: 1073741823 + bgwriter_lru_multiplier: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 10 + bonjour: + - type: Bool + version_from: 90300 + bonjour_name: + - type: String + version_from: 90300 + bytea_output: + - type: Enum + version_from: 90300 + possible_values: + - escape + - hex + check_function_bodies: + - type: Bool + version_from: 90300 + checkpoint_completion_target: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 1 + checkpoint_flush_after: + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 256 + unit: 8kB + checkpoint_segments: + - type: Integer + version_from: 90300 + version_till: 90500 + min_val: 1 + max_val: 2147483647 + checkpoint_timeout: + - type: Integer + version_from: 90300 + version_till: 90600 + min_val: 30 + max_val: 3600 + unit: s + - type: Integer + version_from: 90600 + min_val: 30 + max_val: 86400 + unit: s + checkpoint_warning: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + unit: s + client_connection_check_interval: + - type: Integer + version_from: 140000 + min_val: 0 + max_val: 2147483647 + unit: ms + client_encoding: + - type: String + version_from: 90300 + client_min_messages: + - type: Enum + version_from: 90300 + possible_values: + - debug5 + - debug4 + - debug3 + - debug2 + - debug1 + - log + - notice + - warning + - error + cluster_name: + - type: String + version_from: 90500 + commit_delay: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 100000 + commit_siblings: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 1000 + compute_query_id: + - type: EnumBool + version_from: 140000 + version_till: 150000 + possible_values: + - auto + - type: EnumBool + version_from: 150000 + possible_values: + - auto + - regress + config_file: + - type: String + version_from: 90300 + constraint_exclusion: + - type: EnumBool + version_from: 90300 + possible_values: + - partition + cpu_index_tuple_cost: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 1.79769e+308 + cpu_operator_cost: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 1.79769e+308 + cpu_tuple_cost: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 1.79769e+308 + cursor_tuple_fraction: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 1 + data_directory: + - type: String + version_from: 90300 + data_sync_retry: + - type: Bool + version_from: 90400 + DateStyle: + - type: String + version_from: 90300 + db_user_namespace: + - type: Bool + version_from: 90300 + deadlock_timeout: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 2147483647 + unit: ms + debug_discard_caches: + - type: Integer + version_from: 150000 + min_val: 0 + max_val: 0 + debug_pretty_print: + - type: Bool + version_from: 90300 + debug_print_parse: + - type: Bool + version_from: 90300 + debug_print_plan: + - type: Bool + version_from: 90300 + debug_print_rewritten: + - type: Bool + version_from: 90300 + default_statistics_target: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 10000 + default_table_access_method: + - type: String + version_from: 120000 + default_tablespace: + - type: String + version_from: 90300 + default_text_search_config: + - type: String + version_from: 90300 + default_toast_compression: + - type: Enum + version_from: 140000 + possible_values: + - pglz + - lz4 + default_transaction_deferrable: + - type: Bool + version_from: 90300 + default_transaction_isolation: + - type: Enum + version_from: 90300 + possible_values: + - serializable + - repeatable read + - read committed + - read uncommitted + default_transaction_read_only: + - type: Bool + version_from: 90300 + default_with_oids: + - type: Bool + version_from: 90300 + version_till: 120000 + dynamic_library_path: + - type: String + version_from: 90300 + dynamic_shared_memory_type: + - type: Enum + version_from: 90400 + version_till: 120000 + possible_values: + - posix + - sysv + - mmap + - none + - type: Enum + version_from: 120000 + possible_values: + - posix + - sysv + - mmap + effective_cache_size: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 2147483647 + unit: 8kB + effective_io_concurrency: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 1000 + enable_async_append: + - type: Bool + version_from: 140000 + enable_bitmapscan: + - type: Bool + version_from: 90300 + enable_gathermerge: + - type: Bool + version_from: 100000 + enable_hashagg: + - type: Bool + version_from: 90300 + enable_hashjoin: + - type: Bool + version_from: 90300 + enable_incremental_sort: + - type: Bool + version_from: 130000 + enable_indexonlyscan: + - type: Bool + version_from: 90300 + enable_indexscan: + - type: Bool + version_from: 90300 + enable_material: + - type: Bool + version_from: 90300 + enable_memoize: + - type: Bool + version_from: 150000 + enable_mergejoin: + - type: Bool + version_from: 90300 + enable_nestloop: + - type: Bool + version_from: 90300 + enable_parallel_append: + - type: Bool + version_from: 110000 + enable_parallel_hash: + - type: Bool + version_from: 110000 + enable_partition_pruning: + - type: Bool + version_from: 110000 + enable_partitionwise_aggregate: + - type: Bool + version_from: 110000 + enable_partitionwise_join: + - type: Bool + version_from: 110000 + enable_seqscan: + - type: Bool + version_from: 90300 + enable_sort: + - type: Bool + version_from: 90300 + enable_tidscan: + - type: Bool + version_from: 90300 + escape_string_warning: + - type: Bool + version_from: 90300 + event_source: + - type: String + version_from: 90300 + exit_on_error: + - type: Bool + version_from: 90300 + extension_destdir: + - type: String + version_from: 140000 + external_pid_file: + - type: String + version_from: 90300 + extra_float_digits: + - type: Integer + version_from: 90300 + min_val: -15 + max_val: 3 + force_parallel_mode: + - type: EnumBool + version_from: 90600 + possible_values: + - regress + from_collapse_limit: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 2147483647 + fsync: + - type: Bool + version_from: 90300 + full_page_writes: + - type: Bool + version_from: 90300 + geqo: + - type: Bool + version_from: 90300 + geqo_effort: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 10 + geqo_generations: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + geqo_pool_size: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + geqo_seed: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 1 + geqo_selection_bias: + - type: Real + version_from: 90300 + min_val: 1.5 + max_val: 2 + geqo_threshold: + - type: Integer + version_from: 90300 + min_val: 2 + max_val: 2147483647 + gin_fuzzy_search_limit: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + gin_pending_list_limit: + - type: Integer + version_from: 90500 + min_val: 64 + max_val: 2147483647 + unit: kB + hash_mem_multiplier: + - type: Real + version_from: 130000 + min_val: 1 + max_val: 1000 + hba_file: + - type: String + version_from: 90300 + hot_standby: + - type: Bool + version_from: 90300 + hot_standby_feedback: + - type: Bool + version_from: 90300 + huge_pages: + - type: EnumBool + version_from: 90400 + possible_values: + - try + huge_page_size: + - type: Integer + version_from: 140000 + min_val: 0 + max_val: 2147483647 + unit: kB + ident_file: + - type: String + version_from: 90300 + idle_in_transaction_session_timeout: + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 2147483647 + unit: ms + idle_session_timeout: + - type: Integer + version_from: 140000 + min_val: 0 + max_val: 2147483647 + unit: ms + ignore_checksum_failure: + - type: Bool + version_from: 90300 + ignore_invalid_pages: + - type: Bool + version_from: 130000 + ignore_system_indexes: + - type: Bool + version_from: 90300 + IntervalStyle: + - type: Enum + version_from: 90300 + possible_values: + - postgres + - postgres_verbose + - sql_standard + - iso_8601 + jit: + - type: Bool + version_from: 110000 + jit_above_cost: + - type: Real + version_from: 110000 + min_val: -1 + max_val: 1.79769e+308 + jit_debugging_support: + - type: Bool + version_from: 110000 + jit_dump_bitcode: + - type: Bool + version_from: 110000 + jit_expressions: + - type: Bool + version_from: 110000 + jit_inline_above_cost: + - type: Real + version_from: 110000 + min_val: -1 + max_val: 1.79769e+308 + jit_optimize_above_cost: + - type: Real + version_from: 110000 + min_val: -1 + max_val: 1.79769e+308 + jit_profiling_support: + - type: Bool + version_from: 110000 + jit_provider: + - type: String + version_from: 110000 + jit_tuple_deforming: + - type: Bool + version_from: 110000 + join_collapse_limit: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 2147483647 + krb_caseins_users: + - type: Bool + version_from: 90300 + krb_server_keyfile: + - type: String + version_from: 90300 + krb_srvname: + - type: String + version_from: 90300 + version_till: 90400 + lc_messages: + - type: String + version_from: 90300 + lc_monetary: + - type: String + version_from: 90300 + lc_numeric: + - type: String + version_from: 90300 + lc_time: + - type: String + version_from: 90300 + listen_addresses: + - type: String + version_from: 90300 + local_preload_libraries: + - type: String + version_from: 90300 + lock_timeout: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + unit: ms + lo_compat_privileges: + - type: Bool + version_from: 90300 + log_autovacuum_min_duration: + - type: Integer + version_from: 90300 + min_val: -1 + max_val: 2147483647 + unit: ms + log_checkpoints: + - type: Bool + version_from: 90300 + log_connections: + - type: Bool + version_from: 90300 + log_destination: + - type: String + version_from: 90300 + log_directory: + - type: String + version_from: 90300 + log_disconnections: + - type: Bool + version_from: 90300 + log_duration: + - type: Bool + version_from: 90300 + log_error_verbosity: + - type: Enum + version_from: 90300 + possible_values: + - terse + - default + - verbose + log_executor_stats: + - type: Bool + version_from: 90300 + log_file_mode: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 511 + log_filename: + - type: String + version_from: 90300 + logging_collector: + - type: Bool + version_from: 90300 + log_hostname: + - type: Bool + version_from: 90300 + logical_decoding_work_mem: + - type: Integer + version_from: 130000 + min_val: 64 + max_val: 2147483647 + unit: kB + log_line_prefix: + - type: String + version_from: 90300 + log_lock_waits: + - type: Bool + version_from: 90300 + log_min_duration_sample: + - type: Integer + version_from: 130000 + min_val: -1 + max_val: 2147483647 + unit: ms + log_min_duration_statement: + - type: Integer + version_from: 90300 + min_val: -1 + max_val: 2147483647 + unit: ms + log_min_error_statement: + - type: Enum + version_from: 90300 + possible_values: + - debug5 + - debug4 + - debug3 + - debug2 + - debug1 + - info + - notice + - warning + - error + - log + - fatal + - panic + log_min_messages: + - type: Enum + version_from: 90300 + possible_values: + - debug5 + - debug4 + - debug3 + - debug2 + - debug1 + - info + - notice + - warning + - error + - log + - fatal + - panic + log_parameter_max_length: + - type: Integer + version_from: 130000 + min_val: -1 + max_val: 1073741823 + unit: B + log_parameter_max_length_on_error: + - type: Integer + version_from: 130000 + min_val: -1 + max_val: 1073741823 + unit: B + log_parser_stats: + - type: Bool + version_from: 90300 + log_planner_stats: + - type: Bool + version_from: 90300 + log_recovery_conflict_waits: + - type: Bool + version_from: 140000 + log_replication_commands: + - type: Bool + version_from: 90500 + log_rotation_age: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 35791394 + unit: min + log_rotation_size: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2097151 + unit: kB + log_startup_progress_interval: + - type: Integer + version_from: 150000 + min_val: 0 + max_val: 2147483647 + unit: ms + log_statement: + - type: Enum + version_from: 90300 + possible_values: + - none + - ddl + - mod + - all + log_statement_sample_rate: + - type: Real + version_from: 130000 + min_val: 0 + max_val: 1 + log_statement_stats: + - type: Bool + version_from: 90300 + log_temp_files: + - type: Integer + version_from: 90300 + min_val: -1 + max_val: 2147483647 + unit: kB + log_timezone: + - type: String + version_from: 90300 + log_transaction_sample_rate: + - type: Real + version_from: 120000 + min_val: 0 + max_val: 1 + log_truncate_on_rotation: + - type: Bool + version_from: 90300 + maintenance_io_concurrency: + - type: Integer + version_from: 130000 + min_val: 0 + max_val: 1000 + maintenance_work_mem: + - type: Integer + version_from: 90300 + min_val: 1024 + max_val: 2147483647 + unit: kB + max_connections: + - type: Integer + version_from: 90300 + version_till: 90600 + min_val: 1 + max_val: 8388607 + - type: Integer + version_from: 90600 + min_val: 1 + max_val: 262143 + max_files_per_process: + - type: Integer + version_from: 90300 + version_till: 130000 + min_val: 25 + max_val: 2147483647 + - type: Integer + version_from: 130000 + min_val: 64 + max_val: 2147483647 + max_locks_per_transaction: + - type: Integer + version_from: 90300 + min_val: 10 + max_val: 2147483647 + max_logical_replication_workers: + - type: Integer + version_from: 100000 + min_val: 0 + max_val: 262143 + max_parallel_maintenance_workers: + - type: Integer + version_from: 110000 + min_val: 0 + max_val: 1024 + max_parallel_workers: + - type: Integer + version_from: 100000 + min_val: 0 + max_val: 1024 + max_parallel_workers_per_gather: + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 1024 + max_pred_locks_per_page: + - type: Integer + version_from: 100000 + min_val: 0 + max_val: 2147483647 + max_pred_locks_per_relation: + - type: Integer + version_from: 100000 + min_val: -2147483648 + max_val: 2147483647 + max_pred_locks_per_transaction: + - type: Integer + version_from: 90300 + min_val: 10 + max_val: 2147483647 + max_prepared_transactions: + - type: Integer + version_from: 90300 + version_till: 90600 + min_val: 0 + max_val: 8388607 + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 262143 + max_replication_slots: + - type: Integer + version_from: 90400 + version_till: 90600 + min_val: 0 + max_val: 8388607 + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 262143 + max_slot_wal_keep_size: + - type: Integer + version_from: 130000 + min_val: -1 + max_val: 2147483647 + unit: MB + max_stack_depth: + - type: Integer + version_from: 90300 + min_val: 100 + max_val: 2147483647 + unit: kB + max_standby_archive_delay: + - type: Integer + version_from: 90300 + min_val: -1 + max_val: 2147483647 + unit: ms + max_standby_streaming_delay: + - type: Integer + version_from: 90300 + min_val: -1 + max_val: 2147483647 + unit: ms + max_sync_workers_per_subscription: + - type: Integer + version_from: 100000 + min_val: 0 + max_val: 262143 + max_wal_senders: + - type: Integer + version_from: 90300 + version_till: 90600 + min_val: 0 + max_val: 8388607 + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 262143 + max_wal_size: + - type: Integer + version_from: 90500 + version_till: 100000 + min_val: 2 + max_val: 2147483647 + unit: 16MB + - type: Integer + version_from: 100000 + min_val: 2 + max_val: 2147483647 + unit: MB + max_worker_processes: + - type: Integer + version_from: 90400 + version_till: 90600 + min_val: 1 + max_val: 8388607 + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 262143 + min_dynamic_shared_memory: + - type: Integer + version_from: 140000 + min_val: 0 + max_val: 2147483647 + unit: MB + min_parallel_index_scan_size: + - type: Integer + version_from: 100000 + min_val: 0 + max_val: 715827882 + unit: 8kB + min_parallel_relation_size: + - type: Integer + version_from: 90600 + version_till: 100000 + min_val: 0 + max_val: 715827882 + unit: 8kB + min_parallel_table_scan_size: + - type: Integer + version_from: 100000 + min_val: 0 + max_val: 715827882 + unit: 8kB + min_wal_size: + - type: Integer + version_from: 90500 + version_till: 100000 + min_val: 2 + max_val: 2147483647 + unit: 16MB + - type: Integer + version_from: 100000 + min_val: 2 + max_val: 2147483647 + unit: MB + old_snapshot_threshold: + - type: Integer + version_from: 90600 + min_val: -1 + max_val: 86400 + unit: min + operator_precedence_warning: + - type: Bool + version_from: 90500 + version_till: 140000 + parallel_leader_participation: + - type: Bool + version_from: 110000 + parallel_setup_cost: + - type: Real + version_from: 90600 + min_val: 0 + max_val: 1.79769e+308 + parallel_tuple_cost: + - type: Real + version_from: 90600 + min_val: 0 + max_val: 1.79769e+308 + password_encryption: + - type: Bool + version_from: 90300 + version_till: 100000 + - type: Enum + version_from: 100000 + possible_values: + - md5 + - scram-sha-256 + plan_cache_mode: + - type: Enum + version_from: 120000 + possible_values: + - auto + - force_generic_plan + - force_custom_plan + port: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 65535 + post_auth_delay: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147 + unit: s + pre_auth_delay: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 60 + unit: s + quote_all_identifiers: + - type: Bool + version_from: 90300 + random_page_cost: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 1.79769e+308 + recovery_init_sync_method: + - type: Enum + version_from: 140000 + possible_values: + - fsync + - syncfs + recovery_prefetch: + - type: EnumBool + version_from: 150000 + possible_values: + - try + recursive_worktable_factor: + - type: Real + version_from: 150000 + min_val: 0.001 + max_val: 1000000.0 + remove_temp_files_after_crash: + - type: Bool + version_from: 140000 + replacement_sort_tuples: + - type: Integer + version_from: 90600 + version_till: 110000 + min_val: 0 + max_val: 2147483647 + restart_after_crash: + - type: Bool + version_from: 90300 + row_security: + - type: Bool + version_from: 90500 + search_path: + - type: String + version_from: 90300 + seq_page_cost: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 1.79769e+308 + session_preload_libraries: + - type: String + version_from: 90400 + session_replication_role: + - type: Enum + version_from: 90300 + possible_values: + - origin + - replica + - local + shared_buffers: + - type: Integer + version_from: 90300 + min_val: 16 + max_val: 1073741823 + unit: 8kB + shared_memory_type: + - type: Enum + version_from: 120000 + possible_values: + - sysv + - mmap + shared_preload_libraries: + - type: String + version_from: 90300 + sql_inheritance: + - type: Bool + version_from: 90300 + version_till: 100000 + ssl: + - type: Bool + version_from: 90300 + ssl_ca_file: + - type: String + version_from: 90300 + ssl_cert_file: + - type: String + version_from: 90300 + ssl_ciphers: + - type: String + version_from: 90300 + ssl_crl_dir: + - type: String + version_from: 140000 + ssl_crl_file: + - type: String + version_from: 90300 + ssl_dh_params_file: + - type: String + version_from: 100000 + ssl_ecdh_curve: + - type: String + version_from: 90400 + ssl_key_file: + - type: String + version_from: 90300 + ssl_max_protocol_version: + - type: Enum + version_from: 120000 + possible_values: + - '' + - tlsv1 + - tlsv1.1 + - tlsv1.2 + - tlsv1.3 + ssl_min_protocol_version: + - type: Enum + version_from: 120000 + possible_values: + - tlsv1 + - tlsv1.1 + - tlsv1.2 + - tlsv1.3 + ssl_passphrase_command: + - type: String + version_from: 110000 + ssl_passphrase_command_supports_reload: + - type: Bool + version_from: 110000 + ssl_prefer_server_ciphers: + - type: Bool + version_from: 90400 + ssl_renegotiation_limit: + - type: Integer + version_from: 90300 + version_till: 90500 + min_val: 0 + max_val: 2147483647 + unit: kB + standard_conforming_strings: + - type: Bool + version_from: 90300 + statement_timeout: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + unit: ms + stats_fetch_consistency: + - type: Enum + version_from: 150000 + possible_values: + - none + - cache + - snapshot + stats_temp_directory: + - type: String + version_from: 90300 + version_till: 150000 + superuser_reserved_connections: + - type: Integer + version_from: 90300 + version_till: 90600 + min_val: 0 + max_val: 8388607 + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 262143 + synchronize_seqscans: + - type: Bool + version_from: 90300 + synchronous_commit: + - type: EnumBool + version_from: 90300 + version_till: 90600 + possible_values: + - local + - remote_write + - type: EnumBool + version_from: 90600 + possible_values: + - local + - remote_write + - remote_apply + synchronous_standby_names: + - type: String + version_from: 90300 + syslog_facility: + - type: Enum + version_from: 90300 + possible_values: + - local0 + - local1 + - local2 + - local3 + - local4 + - local5 + - local6 + - local7 + syslog_ident: + - type: String + version_from: 90300 + syslog_sequence_numbers: + - type: Bool + version_from: 90600 + syslog_split_messages: + - type: Bool + version_from: 90600 + tcp_keepalives_count: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + tcp_keepalives_idle: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + unit: s + tcp_keepalives_interval: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + unit: s + tcp_user_timeout: + - type: Integer + version_from: 120000 + min_val: 0 + max_val: 2147483647 + unit: ms + temp_buffers: + - type: Integer + version_from: 90300 + min_val: 100 + max_val: 1073741823 + unit: 8kB + temp_file_limit: + - type: Integer + version_from: 90300 + min_val: -1 + max_val: 2147483647 + unit: kB + temp_tablespaces: + - type: String + version_from: 90300 + TimeZone: + - type: String + version_from: 90300 + timezone_abbreviations: + - type: String + version_from: 90300 + trace_notify: + - type: Bool + version_from: 90300 + trace_recovery_messages: + - type: Enum + version_from: 90300 + possible_values: + - debug5 + - debug4 + - debug3 + - debug2 + - debug1 + - log + - notice + - warning + - error + trace_sort: + - type: Bool + version_from: 90300 + track_activities: + - type: Bool + version_from: 90300 + track_activity_query_size: + - type: Integer + version_from: 90300 + version_till: 110000 + min_val: 100 + max_val: 102400 + - type: Integer + version_from: 110000 + version_till: 130000 + min_val: 100 + max_val: 102400 + unit: B + - type: Integer + version_from: 130000 + min_val: 100 + max_val: 1048576 + unit: B + track_commit_timestamp: + - type: Bool + version_from: 90500 + track_counts: + - type: Bool + version_from: 90300 + track_functions: + - type: Enum + version_from: 90300 + possible_values: + - none + - pl + - all + track_io_timing: + - type: Bool + version_from: 90300 + track_wal_io_timing: + - type: Bool + version_from: 140000 + transaction_deferrable: + - type: Bool + version_from: 90300 + transaction_isolation: + - type: Enum + version_from: 90300 + possible_values: + - serializable + - repeatable read + - read committed + - read uncommitted + transaction_read_only: + - type: Bool + version_from: 90300 + transform_null_equals: + - type: Bool + version_from: 90300 + unix_socket_directories: + - type: String + version_from: 90300 + unix_socket_group: + - type: String + version_from: 90300 + unix_socket_permissions: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 511 + update_process_title: + - type: Bool + version_from: 90300 + vacuum_cleanup_index_scale_factor: + - type: Real + version_from: 110000 + version_till: 140000 + min_val: 0 + max_val: 10000000000.0 + vacuum_cost_delay: + - type: Integer + version_from: 90300 + version_till: 120000 + min_val: 0 + max_val: 100 + unit: ms + - type: Real + version_from: 120000 + min_val: 0 + max_val: 100 + unit: ms + vacuum_cost_limit: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 10000 + vacuum_cost_page_dirty: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 10000 + vacuum_cost_page_hit: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 10000 + vacuum_cost_page_miss: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 10000 + vacuum_defer_cleanup_age: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 1000000 + vacuum_failsafe_age: + - type: Integer + version_from: 140000 + min_val: 0 + max_val: 2100000000 + vacuum_freeze_min_age: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 1000000000 + vacuum_freeze_table_age: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2000000000 + vacuum_multixact_failsafe_age: + - type: Integer + version_from: 140000 + min_val: 0 + max_val: 2100000000 + vacuum_multixact_freeze_min_age: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 1000000000 + vacuum_multixact_freeze_table_age: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2000000000 + wal_buffers: + - type: Integer + version_from: 90300 + min_val: -1 + max_val: 262143 + unit: 8kB + wal_compression: + - type: Bool + version_from: 90500 + version_till: 150000 + - type: EnumBool + version_from: 150000 + possible_values: + - pglz + - lz4 + - zstd + wal_consistency_checking: + - type: String + version_from: 100000 + wal_decode_buffer_size: + - type: Integer + version_from: 150000 + min_val: 65536 + max_val: 1073741823 + unit: B + wal_init_zero: + - type: Bool + version_from: 120000 + wal_keep_segments: + - type: Integer + version_from: 90300 + version_till: 130000 + min_val: 0 + max_val: 2147483647 + wal_keep_size: + - type: Integer + version_from: 130000 + min_val: 0 + max_val: 2147483647 + unit: MB + wal_level: + - type: Enum + version_from: 90300 + version_till: 90400 + possible_values: + - minimal + - archive + - hot_standby + - type: Enum + version_from: 90400 + version_till: 90600 + possible_values: + - minimal + - archive + - hot_standby + - logical + - type: Enum + version_from: 90600 + possible_values: + - minimal + - replica + - logical + wal_log_hints: + - type: Bool + version_from: 90400 + wal_receiver_create_temp_slot: + - type: Bool + version_from: 130000 + wal_receiver_status_interval: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483 + unit: s + wal_receiver_timeout: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + unit: ms + wal_recycle: + - type: Bool + version_from: 120000 + wal_retrieve_retry_interval: + - type: Integer + version_from: 90500 + min_val: 1 + max_val: 2147483647 + unit: ms + wal_sender_timeout: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + unit: ms + wal_skip_threshold: + - type: Integer + version_from: 130000 + min_val: 0 + max_val: 2147483647 + unit: kB + wal_sync_method: + - type: Enum + version_from: 90300 + possible_values: + - fsync + - fdatasync + - open_sync + - open_datasync + wal_writer_delay: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 10000 + unit: ms + wal_writer_flush_after: + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 2147483647 + unit: 8kB + work_mem: + - type: Integer + version_from: 90300 + min_val: 64 + max_val: 2147483647 + unit: kB + xmlbinary: + - type: Enum + version_from: 90300 + possible_values: + - base64 + - hex + xmloption: + - type: Enum + version_from: 90300 + possible_values: + - content + - document + zero_damaged_pages: + - type: Bool + version_from: 90300 +recovery_parameters: + archive_cleanup_command: + - type: String + version_from: 90300 + pause_at_recovery_target: + - type: Bool + version_from: 90300 + version_till: 90500 + primary_conninfo: + - type: String + version_from: 90300 + primary_slot_name: + - type: String + version_from: 90400 + promote_trigger_file: + - type: String + version_from: 120000 + recovery_end_command: + - type: String + version_from: 90300 + recovery_min_apply_delay: + - type: Integer + version_from: 90400 + min_val: 0 + max_val: 2147483647 + unit: ms + recovery_target: + - type: Enum + version_from: 90400 + possible_values: + - immediate + - '' + recovery_target_action: + - type: Enum + version_from: 90500 + possible_values: + - pause + - promote + - shutdown + recovery_target_inclusive: + - type: Bool + version_from: 90300 + recovery_target_lsn: + - type: String + version_from: 100000 + recovery_target_name: + - type: String + version_from: 90400 + recovery_target_time: + - type: String + version_from: 90300 + recovery_target_timeline: + - type: String + version_from: 90300 + recovery_target_xid: + - type: String + version_from: 90300 + restore_command: + - type: String + version_from: 90300 + standby_mode: + - type: Bool + version_from: 90300 + version_till: 120000 + trigger_file: + - type: String + version_from: 90300 + version_till: 120000 + diff --git a/patroni/postgresql/config.py b/patroni/postgresql/config.py index 76a3fb7b..224b48ef 100644 --- a/patroni/postgresql/config.py +++ b/patroni/postgresql/config.py @@ -412,7 +412,8 @@ class ConfigHandler(object): include = self._config.get('custom_conf') or self._postgresql_base_conf_name f.writeline("include '{0}'\n".format(ConfigWriter.escape(include))) for name, value in sorted((configuration).items()): - value = transform_postgresql_parameter_value(self._postgresql.major_version, name, value) + value = transform_postgresql_parameter_value(self._postgresql.major_version, name, value, + self._postgresql.available_gucs) if value is not None and\ (name != 'hba_file' or not self._postgresql.bootstrap.running_custom_bootstrap): f.write_param(name, value) @@ -534,7 +535,8 @@ class ConfigHandler(object): self._passfile_mtime = mtime(self._pgpass) value = self.format_dsn(value) else: - value = transform_recovery_parameter_value(self._postgresql.major_version, name, value) + value = transform_recovery_parameter_value(self._postgresql.major_version, name, value, + self._postgresql.available_gucs) if value is None: continue fd.write_param(name, value) diff --git a/patroni/postgresql/validator.py b/patroni/postgresql/validator.py index 30546444..7da0003a 100644 --- a/patroni/postgresql/validator.py +++ b/patroni/postgresql/validator.py @@ -1,9 +1,13 @@ import abc +from copy import deepcopy import logging +import os +import yaml -from typing import Any, MutableMapping, Optional, Tuple, Union +from typing import Any, Dict, Iterator, List, MutableMapping, Optional, Tuple, Type, Union -from ..collections import CaseInsensitiveDict +from ..collections import CaseInsensitiveDict, CaseInsensitiveSet +from ..exceptions import PatroniException from ..utils import parse_bool, parse_int, parse_real logger = logging.getLogger(__name__) @@ -11,10 +15,20 @@ logger = logging.getLogger(__name__) class _Transformable(abc.ABC): - def __init__(self, version_from: int, version_till: Optional[int]) -> None: + def __init__(self, version_from: int, version_till: Optional[int] = None) -> None: self.__version_from = version_from self.__version_till = version_till + @classmethod + def get_subclasses(cls) -> Iterator[Type['_Transformable']]: + """Recursively get all subclasses of :class:`_Transformable`. + + :yields: each subclass of :class:`_Transformable`. + """ + for subclass in cls.__subclasses__(): + yield from subclass.get_subclasses() + yield subclass + @property def version_from(self) -> int: return self.__version_from @@ -43,8 +57,8 @@ class Bool(_Transformable): class Number(_Transformable): - def __init__(self, version_from: int, version_till: Optional[int], - min_val: Union[int, float], max_val: Union[int, float], unit: Optional[str]) -> None: + def __init__(self, *, version_from: int, version_till: Optional[int] = None, min_val: Union[int, float], + max_val: Union[int, float], unit: Optional[str] = None) -> None: super(Number, self).__init__(version_from, version_till) self.__min_val = min_val self.__max_val = max_val @@ -99,7 +113,8 @@ class Real(Number): class Enum(_Transformable): - def __init__(self, version_from: int, version_till: Optional[int], possible_values: Tuple[str, ...]) -> None: + def __init__(self, *, version_from: int, version_till: Optional[int] = None, + possible_values: Tuple[str, ...]) -> None: super(Enum, self).__init__(version_from, version_till) self.__possible_values = possible_values @@ -128,456 +143,366 @@ class String(_Transformable): # Format: -# key - parameter name -# value - tuple or multiple tuples if something was changing in GUC across postgres versions -parameters = CaseInsensitiveDict({ - 'allow_in_place_tablespaces': Bool(150000, None), - 'allow_system_table_mods': Bool(90300, None), - 'application_name': String(90300, None), - 'archive_command': String(90300, None), - 'archive_library': String(150000, None), - 'archive_mode': ( - Bool(90300, 90500), - EnumBool(90500, None, ('always',)) - ), - 'archive_timeout': Integer(90300, None, 0, 1073741823, 's'), - 'array_nulls': Bool(90300, None), - 'authentication_timeout': Integer(90300, None, 1, 600, 's'), - 'autovacuum': Bool(90300, None), - 'autovacuum_analyze_scale_factor': Real(90300, None, 0, 100, None), - 'autovacuum_analyze_threshold': Integer(90300, None, 0, 2147483647, None), - 'autovacuum_freeze_max_age': Integer(90300, None, 100000, 2000000000, None), - 'autovacuum_max_workers': ( - Integer(90300, 90600, 1, 8388607, None), - Integer(90600, None, 1, 262143, None) - ), - 'autovacuum_multixact_freeze_max_age': Integer(90300, None, 10000, 2000000000, None), - 'autovacuum_naptime': Integer(90300, None, 1, 2147483, 's'), - 'autovacuum_vacuum_cost_delay': ( - Integer(90300, 120000, -1, 100, 'ms'), - Real(120000, None, -1, 100, 'ms') - ), - 'autovacuum_vacuum_cost_limit': Integer(90300, None, -1, 10000, None), - 'autovacuum_vacuum_insert_scale_factor': Real(130000, None, 0, 100, None), - 'autovacuum_vacuum_insert_threshold': Integer(130000, None, -1, 2147483647, None), - 'autovacuum_vacuum_scale_factor': Real(90300, None, 0, 100, None), - 'autovacuum_vacuum_threshold': Integer(90300, None, 0, 2147483647, None), - 'autovacuum_work_mem': Integer(90400, None, -1, 2147483647, 'kB'), - 'backend_flush_after': Integer(90600, None, 0, 256, '8kB'), - 'backslash_quote': EnumBool(90300, None, ('safe_encoding',)), - 'backtrace_functions': String(130000, None), - 'bgwriter_delay': Integer(90300, None, 10, 10000, 'ms'), - 'bgwriter_flush_after': Integer(90600, None, 0, 256, '8kB'), - 'bgwriter_lru_maxpages': ( - Integer(90300, 100000, 0, 1000, None), - Integer(100000, None, 0, 1073741823, None) - ), - 'bgwriter_lru_multiplier': Real(90300, None, 0, 10, None), - 'bonjour': Bool(90300, None), - 'bonjour_name': String(90300, None), - 'bytea_output': Enum(90300, None, ('escape', 'hex')), - 'check_function_bodies': Bool(90300, None), - 'checkpoint_completion_target': Real(90300, None, 0, 1, None), - 'checkpoint_flush_after': Integer(90600, None, 0, 256, '8kB'), - 'checkpoint_segments': Integer(90300, 90500, 1, 2147483647, None), - 'checkpoint_timeout': ( - Integer(90300, 90600, 30, 3600, 's'), - Integer(90600, None, 30, 86400, 's') - ), - 'checkpoint_warning': Integer(90300, None, 0, 2147483647, 's'), - 'client_connection_check_interval': Integer(140000, None, 0, 2147483647, 'ms'), - 'client_encoding': String(90300, None), - 'client_min_messages': Enum(90300, None, ('debug5', 'debug4', 'debug3', 'debug2', - 'debug1', 'log', 'notice', 'warning', 'error')), - 'cluster_name': String(90500, None), - 'commit_delay': Integer(90300, None, 0, 100000, None), - 'commit_siblings': Integer(90300, None, 0, 1000, None), - 'compute_query_id': ( - EnumBool(140000, 150000, ('auto',)), - EnumBool(150000, None, ('auto', 'regress')) - ), - 'config_file': String(90300, None), - 'constraint_exclusion': EnumBool(90300, None, ('partition',)), - 'cpu_index_tuple_cost': Real(90300, None, 0, 1.79769e+308, None), - 'cpu_operator_cost': Real(90300, None, 0, 1.79769e+308, None), - 'cpu_tuple_cost': Real(90300, None, 0, 1.79769e+308, None), - 'cursor_tuple_fraction': Real(90300, None, 0, 1, None), - 'data_directory': String(90300, None), - 'data_sync_retry': Bool(90400, None), - 'DateStyle': String(90300, None), - 'db_user_namespace': Bool(90300, None), - 'deadlock_timeout': Integer(90300, None, 1, 2147483647, 'ms'), - 'debug_discard_caches': Integer(150000, None, 0, 0, None), - 'debug_pretty_print': Bool(90300, None), - 'debug_print_parse': Bool(90300, None), - 'debug_print_plan': Bool(90300, None), - 'debug_print_rewritten': Bool(90300, None), - 'default_statistics_target': Integer(90300, None, 1, 10000, None), - 'default_table_access_method': String(120000, None), - 'default_tablespace': String(90300, None), - 'default_text_search_config': String(90300, None), - 'default_toast_compression': Enum(140000, None, ('pglz', 'lz4')), - 'default_transaction_deferrable': Bool(90300, None), - 'default_transaction_isolation': Enum(90300, None, ('serializable', 'repeatable read', - 'read committed', 'read uncommitted')), - 'default_transaction_read_only': Bool(90300, None), - 'default_with_oids': Bool(90300, 120000), - 'dynamic_library_path': String(90300, None), - 'dynamic_shared_memory_type': ( - Enum(90400, 120000, ('posix', 'sysv', 'mmap', 'none')), - Enum(120000, None, ('posix', 'sysv', 'mmap')) - ), - 'effective_cache_size': Integer(90300, None, 1, 2147483647, '8kB'), - 'effective_io_concurrency': Integer(90300, None, 0, 1000, None), - 'enable_async_append': Bool(140000, None), - 'enable_bitmapscan': Bool(90300, None), - 'enable_gathermerge': Bool(100000, None), - 'enable_hashagg': Bool(90300, None), - 'enable_hashjoin': Bool(90300, None), - 'enable_incremental_sort': Bool(130000, None), - 'enable_indexonlyscan': Bool(90300, None), - 'enable_indexscan': Bool(90300, None), - 'enable_material': Bool(90300, None), - 'enable_memoize': Bool(150000, None), - 'enable_mergejoin': Bool(90300, None), - 'enable_nestloop': Bool(90300, None), - 'enable_parallel_append': Bool(110000, None), - 'enable_parallel_hash': Bool(110000, None), - 'enable_partition_pruning': Bool(110000, None), - 'enable_partitionwise_aggregate': Bool(110000, None), - 'enable_partitionwise_join': Bool(110000, None), - 'enable_seqscan': Bool(90300, None), - 'enable_sort': Bool(90300, None), - 'enable_tidscan': Bool(90300, None), - 'escape_string_warning': Bool(90300, None), - 'event_source': String(90300, None), - 'exit_on_error': Bool(90300, None), - 'extension_destdir': String(140000, None), - 'external_pid_file': String(90300, None), - 'extra_float_digits': Integer(90300, None, -15, 3, None), - 'force_parallel_mode': EnumBool(90600, None, ('regress',)), - 'from_collapse_limit': Integer(90300, None, 1, 2147483647, None), - 'fsync': Bool(90300, None), - 'full_page_writes': Bool(90300, None), - 'geqo': Bool(90300, None), - 'geqo_effort': Integer(90300, None, 1, 10, None), - 'geqo_generations': Integer(90300, None, 0, 2147483647, None), - 'geqo_pool_size': Integer(90300, None, 0, 2147483647, None), - 'geqo_seed': Real(90300, None, 0, 1, None), - 'geqo_selection_bias': Real(90300, None, 1.5, 2, None), - 'geqo_threshold': Integer(90300, None, 2, 2147483647, None), - 'gin_fuzzy_search_limit': Integer(90300, None, 0, 2147483647, None), - 'gin_pending_list_limit': Integer(90500, None, 64, 2147483647, 'kB'), - 'hash_mem_multiplier': Real(130000, None, 1, 1000, None), - 'hba_file': String(90300, None), - 'hot_standby': Bool(90300, None), - 'hot_standby_feedback': Bool(90300, None), - 'huge_pages': EnumBool(90400, None, ('try',)), - 'huge_page_size': Integer(140000, None, 0, 2147483647, 'kB'), - 'ident_file': String(90300, None), - 'idle_in_transaction_session_timeout': Integer(90600, None, 0, 2147483647, 'ms'), - 'idle_session_timeout': Integer(140000, None, 0, 2147483647, 'ms'), - 'ignore_checksum_failure': Bool(90300, None), - 'ignore_invalid_pages': Bool(130000, None), - 'ignore_system_indexes': Bool(90300, None), - 'IntervalStyle': Enum(90300, None, ('postgres', 'postgres_verbose', 'sql_standard', 'iso_8601')), - 'jit': Bool(110000, None), - 'jit_above_cost': Real(110000, None, -1, 1.79769e+308, None), - 'jit_debugging_support': Bool(110000, None), - 'jit_dump_bitcode': Bool(110000, None), - 'jit_expressions': Bool(110000, None), - 'jit_inline_above_cost': Real(110000, None, -1, 1.79769e+308, None), - 'jit_optimize_above_cost': Real(110000, None, -1, 1.79769e+308, None), - 'jit_profiling_support': Bool(110000, None), - 'jit_provider': String(110000, None), - 'jit_tuple_deforming': Bool(110000, None), - 'join_collapse_limit': Integer(90300, None, 1, 2147483647, None), - 'krb_caseins_users': Bool(90300, None), - 'krb_server_keyfile': String(90300, None), - 'krb_srvname': String(90300, 90400), - 'lc_messages': String(90300, None), - 'lc_monetary': String(90300, None), - 'lc_numeric': String(90300, None), - 'lc_time': String(90300, None), - 'listen_addresses': String(90300, None), - 'local_preload_libraries': String(90300, None), - 'lock_timeout': Integer(90300, None, 0, 2147483647, 'ms'), - 'lo_compat_privileges': Bool(90300, None), - 'log_autovacuum_min_duration': Integer(90300, None, -1, 2147483647, 'ms'), - 'log_checkpoints': Bool(90300, None), - 'log_connections': Bool(90300, None), - 'log_destination': String(90300, None), - 'log_directory': String(90300, None), - 'log_disconnections': Bool(90300, None), - 'log_duration': Bool(90300, None), - 'log_error_verbosity': Enum(90300, None, ('terse', 'default', 'verbose')), - 'log_executor_stats': Bool(90300, None), - 'log_file_mode': Integer(90300, None, 0, 511, None), - 'log_filename': String(90300, None), - 'logging_collector': Bool(90300, None), - 'log_hostname': Bool(90300, None), - 'logical_decoding_work_mem': Integer(130000, None, 64, 2147483647, 'kB'), - 'log_line_prefix': String(90300, None), - 'log_lock_waits': Bool(90300, None), - 'log_min_duration_sample': Integer(130000, None, -1, 2147483647, 'ms'), - 'log_min_duration_statement': Integer(90300, None, -1, 2147483647, 'ms'), - 'log_min_error_statement': Enum(90300, None, ('debug5', 'debug4', 'debug3', 'debug2', 'debug1', 'info', - 'notice', 'warning', 'error', 'log', 'fatal', 'panic')), - 'log_min_messages': Enum(90300, None, ('debug5', 'debug4', 'debug3', 'debug2', 'debug1', 'info', - 'notice', 'warning', 'error', 'log', 'fatal', 'panic')), - 'log_parameter_max_length': Integer(130000, None, -1, 1073741823, 'B'), - 'log_parameter_max_length_on_error': Integer(130000, None, -1, 1073741823, 'B'), - 'log_parser_stats': Bool(90300, None), - 'log_planner_stats': Bool(90300, None), - 'log_recovery_conflict_waits': Bool(140000, None), - 'log_replication_commands': Bool(90500, None), - 'log_rotation_age': Integer(90300, None, 0, 35791394, 'min'), - 'log_rotation_size': Integer(90300, None, 0, 2097151, 'kB'), - 'log_startup_progress_interval': Integer(150000, None, 0, 2147483647, 'ms'), - 'log_statement': Enum(90300, None, ('none', 'ddl', 'mod', 'all')), - 'log_statement_sample_rate': Real(130000, None, 0, 1, None), - 'log_statement_stats': Bool(90300, None), - 'log_temp_files': Integer(90300, None, -1, 2147483647, 'kB'), - 'log_timezone': String(90300, None), - 'log_transaction_sample_rate': Real(120000, None, 0, 1, None), - 'log_truncate_on_rotation': Bool(90300, None), - 'maintenance_io_concurrency': Integer(130000, None, 0, 1000, None), - 'maintenance_work_mem': Integer(90300, None, 1024, 2147483647, 'kB'), - 'max_connections': ( - Integer(90300, 90600, 1, 8388607, None), - Integer(90600, None, 1, 262143, None) - ), - 'max_files_per_process': ( - Integer(90300, 130000, 25, 2147483647, None), - Integer(130000, None, 64, 2147483647, None) - ), - 'max_locks_per_transaction': Integer(90300, None, 10, 2147483647, None), - 'max_logical_replication_workers': Integer(100000, None, 0, 262143, None), - 'max_parallel_maintenance_workers': Integer(110000, None, 0, 1024, None), - 'max_parallel_workers': Integer(100000, None, 0, 1024, None), - 'max_parallel_workers_per_gather': Integer(90600, None, 0, 1024, None), - 'max_pred_locks_per_page': Integer(100000, None, 0, 2147483647, None), - 'max_pred_locks_per_relation': Integer(100000, None, -2147483648, 2147483647, None), - 'max_pred_locks_per_transaction': Integer(90300, None, 10, 2147483647, None), - 'max_prepared_transactions': ( - Integer(90300, 90600, 0, 8388607, None), - Integer(90600, None, 0, 262143, None) - ), - 'max_replication_slots': ( - Integer(90400, 90600, 0, 8388607, None), - Integer(90600, None, 0, 262143, None) - ), - 'max_slot_wal_keep_size': Integer(130000, None, -1, 2147483647, 'MB'), - 'max_stack_depth': Integer(90300, None, 100, 2147483647, 'kB'), - 'max_standby_archive_delay': Integer(90300, None, -1, 2147483647, 'ms'), - 'max_standby_streaming_delay': Integer(90300, None, -1, 2147483647, 'ms'), - 'max_sync_workers_per_subscription': Integer(100000, None, 0, 262143, None), - 'max_wal_senders': ( - Integer(90300, 90600, 0, 8388607, None), - Integer(90600, None, 0, 262143, None) - ), - 'max_wal_size': ( - Integer(90500, 100000, 2, 2147483647, '16MB'), - Integer(100000, None, 2, 2147483647, 'MB') - ), - 'max_worker_processes': ( - Integer(90400, 90600, 1, 8388607, None), - Integer(90600, None, 0, 262143, None) - ), - 'min_dynamic_shared_memory': Integer(140000, None, 0, 2147483647, 'MB'), - 'min_parallel_index_scan_size': Integer(100000, None, 0, 715827882, '8kB'), - 'min_parallel_relation_size': Integer(90600, 100000, 0, 715827882, '8kB'), - 'min_parallel_table_scan_size': Integer(100000, None, 0, 715827882, '8kB'), - 'min_wal_size': ( - Integer(90500, 100000, 2, 2147483647, '16MB'), - Integer(100000, None, 2, 2147483647, 'MB') - ), - 'old_snapshot_threshold': Integer(90600, None, -1, 86400, 'min'), - 'operator_precedence_warning': Bool(90500, 140000), - 'parallel_leader_participation': Bool(110000, None), - 'parallel_setup_cost': Real(90600, None, 0, 1.79769e+308, None), - 'parallel_tuple_cost': Real(90600, None, 0, 1.79769e+308, None), - 'password_encryption': ( - Bool(90300, 100000), - Enum(100000, None, ('md5', 'scram-sha-256')) - ), - 'plan_cache_mode': Enum(120000, None, ('auto', 'force_generic_plan', 'force_custom_plan')), - 'port': Integer(90300, None, 1, 65535, None), - 'post_auth_delay': Integer(90300, None, 0, 2147, 's'), - 'pre_auth_delay': Integer(90300, None, 0, 60, 's'), - 'quote_all_identifiers': Bool(90300, None), - 'random_page_cost': Real(90300, None, 0, 1.79769e+308, None), - 'recovery_init_sync_method': Enum(140000, None, ('fsync', 'syncfs')), - 'recovery_prefetch': EnumBool(150000, None, ('try',)), - 'recursive_worktable_factor': Real(150000, None, 0.001, 1e+06, None), - 'remove_temp_files_after_crash': Bool(140000, None), - 'replacement_sort_tuples': Integer(90600, 110000, 0, 2147483647, None), - 'restart_after_crash': Bool(90300, None), - 'row_security': Bool(90500, None), - 'search_path': String(90300, None), - 'seq_page_cost': Real(90300, None, 0, 1.79769e+308, None), - 'session_preload_libraries': String(90400, None), - 'session_replication_role': Enum(90300, None, ('origin', 'replica', 'local')), - 'shared_buffers': Integer(90300, None, 16, 1073741823, '8kB'), - 'shared_memory_type': Enum(120000, None, ('sysv', 'mmap')), - 'shared_preload_libraries': String(90300, None), - 'sql_inheritance': Bool(90300, 100000), - 'ssl': Bool(90300, None), - 'ssl_ca_file': String(90300, None), - 'ssl_cert_file': String(90300, None), - 'ssl_ciphers': String(90300, None), - 'ssl_crl_dir': String(140000, None), - 'ssl_crl_file': String(90300, None), - 'ssl_dh_params_file': String(100000, None), - 'ssl_ecdh_curve': String(90400, None), - 'ssl_key_file': String(90300, None), - 'ssl_max_protocol_version': Enum(120000, None, ('', 'tlsv1', 'tlsv1.1', 'tlsv1.2', 'tlsv1.3')), - 'ssl_min_protocol_version': Enum(120000, None, ('tlsv1', 'tlsv1.1', 'tlsv1.2', 'tlsv1.3')), - 'ssl_passphrase_command': String(110000, None), - 'ssl_passphrase_command_supports_reload': Bool(110000, None), - 'ssl_prefer_server_ciphers': Bool(90400, None), - 'ssl_renegotiation_limit': Integer(90300, 90500, 0, 2147483647, 'kB'), - 'standard_conforming_strings': Bool(90300, None), - 'statement_timeout': Integer(90300, None, 0, 2147483647, 'ms'), - 'stats_fetch_consistency': Enum(150000, None, ('none', 'cache', 'snapshot')), - 'stats_temp_directory': String(90300, 150000), - 'superuser_reserved_connections': ( - Integer(90300, 90600, 0, 8388607, None), - Integer(90600, None, 0, 262143, None) - ), - 'synchronize_seqscans': Bool(90300, None), - 'synchronous_commit': ( - EnumBool(90300, 90600, ('local', 'remote_write')), - EnumBool(90600, None, ('local', 'remote_write', 'remote_apply')) - ), - 'synchronous_standby_names': String(90300, None), - 'syslog_facility': Enum(90300, None, ('local0', 'local1', 'local2', 'local3', - 'local4', 'local5', 'local6', 'local7')), - 'syslog_ident': String(90300, None), - 'syslog_sequence_numbers': Bool(90600, None), - 'syslog_split_messages': Bool(90600, None), - 'tcp_keepalives_count': Integer(90300, None, 0, 2147483647, None), - 'tcp_keepalives_idle': Integer(90300, None, 0, 2147483647, 's'), - 'tcp_keepalives_interval': Integer(90300, None, 0, 2147483647, 's'), - 'tcp_user_timeout': Integer(120000, None, 0, 2147483647, 'ms'), - 'temp_buffers': Integer(90300, None, 100, 1073741823, '8kB'), - 'temp_file_limit': Integer(90300, None, -1, 2147483647, 'kB'), - 'temp_tablespaces': String(90300, None), - 'TimeZone': String(90300, None), - 'timezone_abbreviations': String(90300, None), - 'trace_notify': Bool(90300, None), - 'trace_recovery_messages': Enum(90300, None, ('debug5', 'debug4', 'debug3', 'debug2', - 'debug1', 'log', 'notice', 'warning', 'error')), - 'trace_sort': Bool(90300, None), - 'track_activities': Bool(90300, None), - 'track_activity_query_size': ( - Integer(90300, 110000, 100, 102400, None), - Integer(110000, 130000, 100, 102400, 'B'), - Integer(130000, None, 100, 1048576, 'B') - ), - 'track_commit_timestamp': Bool(90500, None), - 'track_counts': Bool(90300, None), - 'track_functions': Enum(90300, None, ('none', 'pl', 'all')), - 'track_io_timing': Bool(90300, None), - 'track_wal_io_timing': Bool(140000, None), - 'transaction_deferrable': Bool(90300, None), - 'transaction_isolation': Enum(90300, None, ('serializable', 'repeatable read', - 'read committed', 'read uncommitted')), - 'transaction_read_only': Bool(90300, None), - 'transform_null_equals': Bool(90300, None), - 'unix_socket_directories': String(90300, None), - 'unix_socket_group': String(90300, None), - 'unix_socket_permissions': Integer(90300, None, 0, 511, None), - 'update_process_title': Bool(90300, None), - 'vacuum_cleanup_index_scale_factor': Real(110000, 140000, 0, 1e+10, None), - 'vacuum_cost_delay': ( - Integer(90300, 120000, 0, 100, 'ms'), - Real(120000, None, 0, 100, 'ms') - ), - 'vacuum_cost_limit': Integer(90300, None, 1, 10000, None), - 'vacuum_cost_page_dirty': Integer(90300, None, 0, 10000, None), - 'vacuum_cost_page_hit': Integer(90300, None, 0, 10000, None), - 'vacuum_cost_page_miss': Integer(90300, None, 0, 10000, None), - 'vacuum_defer_cleanup_age': Integer(90300, None, 0, 1000000, None), - 'vacuum_failsafe_age': Integer(140000, None, 0, 2100000000, None), - 'vacuum_freeze_min_age': Integer(90300, None, 0, 1000000000, None), - 'vacuum_freeze_table_age': Integer(90300, None, 0, 2000000000, None), - 'vacuum_multixact_failsafe_age': Integer(140000, None, 0, 2100000000, None), - 'vacuum_multixact_freeze_min_age': Integer(90300, None, 0, 1000000000, None), - 'vacuum_multixact_freeze_table_age': Integer(90300, None, 0, 2000000000, None), - 'wal_buffers': Integer(90300, None, -1, 262143, '8kB'), - 'wal_compression': ( - Bool(90500, 150000), - EnumBool(150000, None, ('pglz', 'lz4', 'zstd')) - ), - 'wal_consistency_checking': String(100000, None), - 'wal_decode_buffer_size': Integer(150000, None, 65536, 1073741823, 'B'), - 'wal_init_zero': Bool(120000, None), - 'wal_keep_segments': Integer(90300, 130000, 0, 2147483647, None), - 'wal_keep_size': Integer(130000, None, 0, 2147483647, 'MB'), - 'wal_level': ( - Enum(90300, 90400, ('minimal', 'archive', 'hot_standby')), - Enum(90400, 90600, ('minimal', 'archive', 'hot_standby', 'logical')), - Enum(90600, None, ('minimal', 'replica', 'logical')) - ), - 'wal_log_hints': Bool(90400, None), - 'wal_receiver_create_temp_slot': Bool(130000, None), - 'wal_receiver_status_interval': Integer(90300, None, 0, 2147483, 's'), - 'wal_receiver_timeout': Integer(90300, None, 0, 2147483647, 'ms'), - 'wal_recycle': Bool(120000, None), - 'wal_retrieve_retry_interval': Integer(90500, None, 1, 2147483647, 'ms'), - 'wal_sender_timeout': Integer(90300, None, 0, 2147483647, 'ms'), - 'wal_skip_threshold': Integer(130000, None, 0, 2147483647, 'kB'), - 'wal_sync_method': Enum(90300, None, ('fsync', 'fdatasync', 'open_sync', 'open_datasync')), - 'wal_writer_delay': Integer(90300, None, 1, 10000, 'ms'), - 'wal_writer_flush_after': Integer(90600, None, 0, 2147483647, '8kB'), - 'work_mem': Integer(90300, None, 64, 2147483647, 'kB'), - 'xmlbinary': Enum(90300, None, ('base64', 'hex')), - 'xmloption': Enum(90300, None, ('content', 'document')), - 'zero_damaged_pages': Bool(90300, None) -}) +# key - parameter name +# value - variable length tuple of `_Transformable` objects. Each object in the tuple represents a different +# validation of the GUC across postgres versions. If a GUC validation has never changed over time, then it will +# have a single object in the tuple. For example, `password_encryption` used to be a boolean GUC up to Postgres +# 10, at which point it started being an enum. In that case the value of `password_encryption` would be a tuple +# of 2 `_Transformable` objects (`Bool` and `Enum`, respectively), each one reprensenting a different +# validation rule. +parameters = CaseInsensitiveDict() +recovery_parameters = CaseInsensitiveDict() -recovery_parameters = CaseInsensitiveDict({ - 'archive_cleanup_command': String(90300, None), - 'pause_at_recovery_target': Bool(90300, 90500), - 'primary_conninfo': String(90300, None), - 'primary_slot_name': String(90400, None), - 'promote_trigger_file': String(120000, None), - 'recovery_end_command': String(90300, None), - 'recovery_min_apply_delay': Integer(90400, None, 0, 2147483647, 'ms'), - 'recovery_target': Enum(90400, None, ('immediate', '')), - 'recovery_target_action': Enum(90500, None, ('pause', 'promote', 'shutdown')), - 'recovery_target_inclusive': Bool(90300, None), - 'recovery_target_lsn': String(100000, None), - 'recovery_target_name': String(90400, None), - 'recovery_target_time': String(90300, None), - 'recovery_target_timeline': String(90300, None), - 'recovery_target_xid': String(90300, None), - 'restore_command': String(90300, None), - 'standby_mode': Bool(90300, 120000), - 'trigger_file': String(90300, 120000) -}) +class ValidatorFactoryNoType(PatroniException): + """Raised when a validator spec misses a type.""" -def _transform_parameter_value(validators: MutableMapping[str, Union[_Transformable, Tuple[_Transformable, ...]]], - version: int, name: str, value: Any) -> Optional[Any]: - name_validators = validators.get(name) - if name_validators: - for validator in (name_validators if isinstance(name_validators, tuple) else (name_validators,)): +class ValidatorFactoryInvalidType(PatroniException): + """Raised when a validator spec contains an invalid type.""" + + +class ValidatorFactoryInvalidSpec(PatroniException): + """Raised when a validator spec contains an invalid set of attributes.""" + + +class ValidatorFactory: + """Factory class used to build Patroni validator objects based on the given specs.""" + + TYPES: Dict[str, Type[_Transformable]] = {cls.__name__: cls for cls in _Transformable.get_subclasses()} + + def __new__(cls, validator: Dict[str, Any]) -> _Transformable: + """Parse a given Postgres GUC *validator* into the corresponding Patroni validator object. + + :param validator: a validator spec for a given parameter. It usually comes from a parsed YAML file. + + :returns: the Patroni validator object that corresponds to the specification found in *validator*. + + :raises :class:`ValidatorFactoryNoType`: if *validator* contains no ``type`` key. + :raises :class:`ValidatorFactoryInvalidType`: if ``type`` key from *validator* contains an invalid value. + :raises :class:`ValidatorFactoryInvalidSpec`: if *validator* contains an invalid set of attributes for the + given ``type``. + + :Example: + + If a given validator was defined as follows in the YAML file: + + ```yaml + - type: String + version_from: 90300 + version_till: null + ``` + + Then this method would receive *validator* as: + + ```python + { + 'type': 'String', + 'version_from': 90300, + 'version_till': None + } + ``` + + And this method would return a :class:`String`: + + ```python + String(90300, None) + ``` + """ + validator = deepcopy(validator) + try: + type_ = validator.pop('type') + except KeyError as exc: + raise ValidatorFactoryNoType('Validator contains no type.') from exc + + if type_ not in cls.TYPES: + raise ValidatorFactoryInvalidType(f'Unexpected validator type: `{type_}`.') + + for key, value in validator.items(): + # :func:`_transform_parameter_value` expects :class:`tuple` instead of :class:`list` + if isinstance(value, list): + tmp_value: List[Any] = value + validator[key] = tuple(tmp_value) + + try: + return cls.TYPES[type_](**validator) + except Exception as exc: + raise ValidatorFactoryInvalidSpec( + f'Failed to parse `{type_}` validator (`{validator}`): `{str(exc)}`.') from exc + + +def _get_postgres_guc_validators(config: Dict[str, Any], parameter: str) -> Tuple[_Transformable, ...]: + """Get all validators of *parameter* from *config*. + + Loop over all validators specs of *parameter* and return them parsed as Patroni validators. + + :param config: Python object corresponding to an YAML file, with values of either ``parameters`` or + ``recovery_parameters`` key. + :param parameter: name of the parameter found under *config* which validators should be parsed and returned. + + :rtype: yields any exception that is faced while parsing a validator spec into a Patroni validator object. + """ + validators: List[_Transformable] = [] + for validator_spec in config.get(parameter, []): + try: + validator = ValidatorFactory(validator_spec) + validators.append(validator) + except (ValidatorFactoryNoType, ValidatorFactoryInvalidType, ValidatorFactoryInvalidSpec) as exc: + logger.warning('Faced an issue while parsing a validator for parameter `%s`: `%r`', parameter, exc) + + return tuple(validators) + + +class InvalidGucValidatorsFile(PatroniException): + """Raised when reading or parsing of a YAML file faces an issue.""" + + +def _read_postgres_gucs_validators_file(file: str) -> Dict[str, Any]: + """Read an YAML file and return the corresponding Python object. + + :param file: path to the file to be read. It is expected to be encoded with ``UTF-8``, and to be a YAML document. + + :returns: the YAML content parsed into a Python object. If any issue is faced while reading/parsing the file, then + return ``None``. + + :raises :class:`InvalidGucValidatorsFile`: if faces an issue while reading or parsing *file*. + """ + try: + with open(file, encoding='UTF-8') as stream: + return yaml.safe_load(stream) + except Exception as exc: + raise InvalidGucValidatorsFile( + f'Unexpected issue while reading parameters file `{file}`: `{str(exc)}`.') from exc + + +def _load_postgres_gucs_validators() -> None: + """Load all Postgres GUC validators from YAML files. + + Recursively walk through ``available_parameters`` directory and load validators of each found YAML file into + ``parameters`` and/or ``recovery_parameters`` variables. + + Walk through directories in top-down fashion and for each of them: + * Sort files by name; + * Load validators from YAML files that were found. + + Any problem faced while reading or parsing files will be logged as a ``WARNING`` by the child function, and the + corresponding file or validator will be ignored. + + By default Patroni only ships the file ``0_postgres.yml``, which contains Community Postgres GUCs validators, but + that behavior can be extended. For example: if a vendor wants to add GUC validators to Patroni for covering a custom + Postgres build, then they can create their custom YAML files under ``available_parameters`` directory. + + Each YAML file may contain either or both of these root attributes, here called sections: + * ``parameters``: general GUCs that would be written to ``postgresql.conf``; + * ``recovery_parameters``: recovery related GUCs that would be written to ``recovery.conf`` (Patroni later + writes them to ``postgresql.conf`` if running PG 12 and above). + + Then, each of these sections, if specified, may contain one or more attributes with the following structure: + * key: the name of a GUC; + * value: a list of validators. Each item in the list must contain a ``type`` attribute, which must be one among: + * ``Bool``; or + * ``Integer``; or + * ``Real``; or + * ``Enum``; or + * ``EnumBool``; or + * ``String``. + + Besides the ``type`` attribute, it should also contain all the required attributes as per the corresponding + class in this module. + + .. seealso:: + * :class:`Bool`; + * :class:`Integer`; + * :class:`Real`; + * :class:`Enum`; + * :class:`EnumBool`; + * :class:`String`. + + :Example: + + This is a sample content for an YAML file based on Postgres GUCs, showing each of the supported types and + sections: + + ```yaml + parameters: + archive_command: + - type: String + version_from: 90300 + version_till: null + archive_mode: + - type: Bool + version_from: 90300 + version_till: 90500 + - type: EnumBool + version_from: 90500 + version_till: null + possible_values: + - always + archive_timeout: + - type: Integer + version_from: 90300 + version_till: null + min_val: 0 + max_val: 1073741823 + unit: s + autovacuum_vacuum_cost_delay: + - type: Integer + version_from: 90300 + version_till: 120000 + min_val: -1 + max_val: 100 + unit: ms + - type: Real + version_from: 120000 + version_till: null + min_val: -1 + max_val: 100 + unit: ms + client_min_messages: + - type: Enum + version_from: 90300 + version_till: null + possible_values: + - debug5 + - debug4 + - debug3 + - debug2 + - debug1 + - log + - notice + - warning + - error + recovery_parameters: + archive_cleanup_command: + - type: String + version_from: 90300 + version_till: null + ``` + """ + conf_dir = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'available_parameters', + ) + yaml_files: List[str] = [] + + for root, _, files in os.walk(conf_dir): + for file in sorted(files): + full_path = os.path.join(root, file) + if file.lower().endswith(('.yml', '.yaml')): + yaml_files.append(full_path) + else: + logger.info('Ignored a non-YAML file found under `available_parameters` directory: `%s`.', full_path) + + for file in yaml_files: + try: + config: Dict[str, Any] = _read_postgres_gucs_validators_file(file) + except InvalidGucValidatorsFile as exc: + logger.warning(str(exc)) + continue + + logger.debug(f'Parsing validators from file `{file}`.') + + mapping = { + 'parameters': parameters, + 'recovery_parameters': recovery_parameters, + } + + for section in ['parameters', 'recovery_parameters']: + section_var = mapping[section] + + config_section = config.get(section, {}) + for parameter in config_section.keys(): + section_var[parameter] = _get_postgres_guc_validators(config_section, parameter) + + +_load_postgres_gucs_validators() + + +def _transform_parameter_value(validators: MutableMapping[str, Tuple[_Transformable, ...]], + version: int, name: str, value: Any, + available_gucs: CaseInsensitiveSet) -> Optional[Any]: + """Validate *value* of GUC *name* for Postgres *version* using defined *validators* and *available_gucs*. + + :param validators: a dictionary of all GUCs across all Postgres versions. Each key is the name of a Postgres GUC, + and the corresponding value is a variable length tuple of :class:`_Transformable`. Each item is a validation + rule for the GUC for a given range of Postgres versions. Should either contain recovery GUCs or general GUCs, + not both. + :param version: Postgres version to validate the GUC against. + :param name: name of the Postgres GUC. + :param value: value of the Postgres GUC. + :param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres + GUC. Used for a couple purposes: + * Disallow writing GUCs to ``postgresql.conf`` (or ``recovery.conf``) that does not exist in Postgres *version*; + * Avoid ignoring GUC *name* if it does not have a validator in *validators*, but is a valid GUC in Postgres + *version*. + + :returns: the return value may be one among: + * *value* transformed to the expected format for GUC *name* in Postgres *version*, if *name* is present in + *available_gucs* and has a validator in *validators* for the corresponding Postgres *version*; or + * The own *value* if *name* is present in *available_gucs* but not in *validators*; or + * ``None`` if *name* is not present in *available_gucs*. + """ + if name in available_gucs: + for validator in validators.get(name, ()) or (): if version >= validator.version_from and\ (validator.version_till is None or version < validator.version_till): return validator.transform(name, value) + # Ideally we should have a validator in *validators*. However, if none is available, we will not discard a + # setting that exists in Postgres *version*, but rather allow the value with no validation. + return value logger.warning('Removing unexpected parameter=%s value=%s from the config', name, value) -def transform_postgresql_parameter_value(version: int, name: str, value: Any) -> Optional[Any]: - if '.' in name: +def transform_postgresql_parameter_value(version: int, name: str, value: Any, + available_gucs: CaseInsensitiveSet) -> Optional[Any]: + """Validate *value* of GUC *name* for Postgres *version* using ``parameters`` and *available_gucs*. + + :param version: Postgres version to validate the GUC against. + :param name: name of the Postgres GUC. + :param value: value of the Postgres GUC. + :param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres + GUC. Used for a couple purposes: + * Disallow writing GUCs to ``postgresql.conf`` that does not exist in Postgres *version*; + * Avoid ignoring GUC *name* if it does not have a validator in ``parameters``, but is a valid GUC in Postgres + *version*. + + :returns: The return value may be one among + * The original *value* if *name* seems to be an extension GUC (contains a period '.'); or + * ``None`` if **name** is a recovery GUC; or + * *value* transformed to the expected format for GUC *name* in Postgres *version* using validators defined in + ``parameters``. Can also return ``None``. See :func:`_transform_parameter_value`. + """ + if '.' in name and name not in parameters: + # likely an extension GUC, so just return as it is. Otherwise, if `name` is in `parameters`, it's likely a + # namespaced GUC from a custom Postgres build, so we treat that over the usual validation means. return value if name in recovery_parameters: return None - return _transform_parameter_value(parameters, version, name, value) + return _transform_parameter_value(parameters, version, name, value, available_gucs) -def transform_recovery_parameter_value(version: int, name: str, value: Any) -> Optional[Any]: - return _transform_parameter_value(recovery_parameters, version, name, value) +def transform_recovery_parameter_value(version: int, name: str, value: Any, + available_gucs: CaseInsensitiveSet) -> Optional[Any]: + """Validate *value* of GUC *name* for Postgres *version* using ``recovery_parameters`` and *available_gucs*. + + :param version: Postgres version to validate the recovery GUC against. + :param name: name of the Postgres recovery GUC. + :param value: value of the Postgres recovery GUC. + :param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres + GUC. Used for a couple purposes: + * Disallow writing GUCs to ``recovery.conf`` (or ``postgresql.conf`` depending on *version*), that does not + exist in Postgres *version*; + * Avoid ignoring recovery GUC *name* if it does not have a validator in ``recovery_parameters``, but is a valid + GUC in Postgres *version*. + + :returns: *value* transformed to the expected format for recovery GUC *name* in Postgres *version* using validators + defined in ``recovery_parameters``. It can also return ``None``. See :func:`_transform_parameter_value`. + """ + # Recovery settings are not present in ``postgres --describe-config`` output of Postgres <= 11. In that case we + # just pass down the list of settings defined in Patroni validators so :func:`_transform_parameter_value` will not + # discard the recovery GUCs when running Postgres <= 11. + # NOTE: At the moment this change was done Postgres 11 was almost EOL, and had been likely extensively used with + # Patroni, so we should be able to rely solely on Patroni validators as the source of truth. + return _transform_parameter_value( + recovery_parameters, version, name, value, + available_gucs if version >= 120000 else CaseInsensitiveSet(recovery_parameters.keys())) diff --git a/setup.py b/setup.py index 1c0d3b3b..bff90cd4 100644 --- a/setup.py +++ b/setup.py @@ -157,7 +157,10 @@ def setup_package(version): long_description=read('README.rst'), classifiers=CLASSIFIERS, packages=find_packages(exclude=['tests', 'tests.*']), - package_data={MAIN_PACKAGE: ["*.json"]}, + package_data={MAIN_PACKAGE: [ + "postgresql/available_parameters/*.yml", + "postgresql/available_parameters/*.yaml", + ]}, install_requires=install_requires, extras_require=EXTRAS_REQUIRE, cmdclass=cmdclass, diff --git a/tests/__init__.py b/tests/__init__.py index 838c2636..03598ae1 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -3,7 +3,7 @@ import os import shutil import unittest -from mock import Mock, patch +from mock import Mock, PropertyMock, patch import urllib3 @@ -19,6 +19,15 @@ class SleepException(Exception): pass +mock_available_gucs = PropertyMock(return_value={ + 'cluster_name', 'constraint_exclusion', 'force_parallel_mode', 'hot_standby', 'listen_addresses', 'max_connections', + 'max_locks_per_transaction', 'max_prepared_transactions', 'max_replication_slots', 'max_stack_depth', + 'max_wal_senders', 'max_worker_processes', 'port', 'search_path', 'shared_preload_libraries', + 'stats_temp_directory', 'synchronous_standby_names', 'track_commit_timestamp', 'unix_socket_directories', + 'vacuum_cost_delay', 'vacuum_cost_limit', 'wal_keep_size', 'wal_level', 'wal_log_hints', 'zero_damaged_pages', +}) + + class MockResponse(object): def __init__(self, status_code=200): diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index bdc284e5..4c7c0fcc 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -9,12 +9,13 @@ from patroni.postgresql.bootstrap import Bootstrap from patroni.postgresql.cancellable import CancellableSubprocess from patroni.postgresql.config import ConfigHandler -from . import psycopg_connect, BaseTestPostgresql +from . import psycopg_connect, BaseTestPostgresql, mock_available_gucs @patch('subprocess.call', Mock(return_value=0)) @patch('patroni.psycopg.connect', psycopg_connect) @patch('os.rename', Mock()) +@patch.object(Postgresql, 'available_gucs', mock_available_gucs) class TestBootstrap(BaseTestPostgresql): @patch('patroni.postgresql.CallbackExecutor', Mock()) diff --git a/tests/test_patroni.py b/tests/test_patroni.py index cb40b232..d5c76c8b 100644 --- a/tests/test_patroni.py +++ b/tests/test_patroni.py @@ -67,6 +67,7 @@ class TestPatroni(unittest.TestCase): @patch.object(etcd.Client, 'read', etcd_read) @patch.object(Thread, 'start', Mock()) @patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379'])) + @patch.object(Postgresql, '_get_gucs', Mock(return_value={'foo': True, 'bar': True})) def setUp(self): self._handlers = logging.getLogger().handlers[:] RestApiServer._BaseServer__is_shut_down = Mock() @@ -90,6 +91,7 @@ class TestPatroni(unittest.TestCase): @patch.object(etcd.Client, 'delete', Mock()) @patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379'])) @patch.object(Thread, 'join', Mock()) + @patch.object(Postgresql, '_get_gucs', Mock(return_value={'foo': True, 'bar': True})) def test_patroni_patroni_main(self): with patch('subprocess.call', Mock(return_value=1)): with patch.object(Patroni, 'run', Mock(side_effect=SleepException)): diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py index f7c0c155..c759cde0 100644 --- a/tests/test_postgresql.py +++ b/tests/test_postgresql.py @@ -10,6 +10,7 @@ from mock import Mock, MagicMock, PropertyMock, patch, mock_open import patroni.psycopg as psycopg from patroni.async_executor import CriticalTask +from patroni.collections import CaseInsensitiveSet from patroni.config import GlobalConfig from patroni.dcs import RemoteMember from patroni.exceptions import PostgresConnectionException, PatroniException @@ -17,10 +18,14 @@ from patroni.postgresql import Postgresql, STATE_REJECT, STATE_NO_RESPONSE from patroni.postgresql.bootstrap import Bootstrap from patroni.postgresql.callback_executor import CallbackAction from patroni.postgresql.postmaster import PostmasterProcess +from patroni.postgresql.validator import (ValidatorFactoryNoType, ValidatorFactoryInvalidType, + ValidatorFactoryInvalidSpec, ValidatorFactory, InvalidGucValidatorsFile, + _get_postgres_guc_validators, _read_postgres_gucs_validators_file, + _load_postgres_gucs_validators, Bool, Integer, Real, Enum, EnumBool, String) from patroni.utils import RetryFailedError from threading import Thread, current_thread -from . import BaseTestPostgresql, MockCursor, MockPostmaster, psycopg_connect +from . import BaseTestPostgresql, MockCursor, MockPostmaster, psycopg_connect, mock_available_gucs mtime_ret = {} @@ -91,6 +96,7 @@ Data page checksum version: 0 @patch('subprocess.call', Mock(return_value=0)) @patch('patroni.psycopg.connect', psycopg_connect) +@patch.object(Postgresql, 'available_gucs', mock_available_gucs) class TestPostgresql(BaseTestPostgresql): @patch('subprocess.call', Mock(return_value=0)) @@ -98,6 +104,7 @@ class TestPostgresql(BaseTestPostgresql): @patch('patroni.postgresql.CallbackExecutor', Mock()) @patch.object(Postgresql, 'get_major_version', Mock(return_value=140000)) @patch.object(Postgresql, 'is_running', Mock(return_value=True)) + @patch.object(Postgresql, 'available_gucs', mock_available_gucs) def setUp(self): super(TestPostgresql, self).setUp() self.p.config.write_postgresql_conf() @@ -739,3 +746,212 @@ class TestPostgresql(BaseTestPostgresql): @patch.object(Postgresql, '_cluster_info_state_get', Mock(return_value=True)) def test_handle_parameter_change(self): self.p.handle_parameter_change() + + def test_validator_factory(self): + # validator with no type + validator = { + 'version_from': 90300, + 'version_till': None, + } + with self.assertRaises(ValidatorFactoryNoType) as e: + ValidatorFactory(validator) + self.assertEqual(str(e.exception), 'Validator contains no type.') + + # validator with invalid type + validator = { + 'type': 'Random', + 'version_from': 90300, + 'version_till': None, + } + with self.assertRaises(ValidatorFactoryInvalidType) as e: + ValidatorFactory(validator) + self.assertEqual(str(e.exception), f'Unexpected validator type: `{validator["type"]}`.') + + # validator with missing attributes + validator = { + 'type': 'Integer', + 'version_from': 90300, + 'min_val': 0, + } + with self.assertRaises(ValidatorFactoryInvalidSpec) as e: + ValidatorFactory(validator) + type_ = validator.pop('type') + self.assertRegex( + str(e.exception), + rf"Failed to parse `{type_}` validator \(`{validator}`\): `(Number\.)?__init__\(\) missing 1 " + "required keyword-only argument: 'max_val'`." + ) + + # valid validators + # Bool + validator = { + 'type': 'Bool', + 'version_from': 90300, + 'version_till': None, + } + ret = ValidatorFactory(validator) + self.assertIsInstance(ret, Bool) + self.assertEqual( + ret.__dict__, + Bool(version_from=validator['version_from'], version_till=validator['version_till']).__dict__, + ) + + # Integer + validator = { + 'type': 'Integer', + 'version_from': 90300, + 'version_till': None, + 'min_val': 1, + 'max_val': 100, + 'unit': None, + } + ret = ValidatorFactory(validator) + self.assertIsInstance(ret, Integer) + self.assertEqual( + ret.__dict__, + Integer(version_from=validator['version_from'], version_till=validator['version_till'], + min_val=validator['min_val'], max_val=validator['max_val'], unit=validator['unit']).__dict__, + ) + + # Real + validator = { + 'type': 'Real', + 'version_from': 90300, + 'version_till': None, + 'min_val': 1.0, + 'max_val': 100.0, + 'unit': None, + } + ret = ValidatorFactory(validator) + self.assertIsInstance(ret, Real) + self.assertEqual( + ret.__dict__, + Real(version_from=validator['version_from'], version_till=validator['version_till'], + min_val=validator['min_val'], max_val=validator['max_val'], unit=validator['unit']).__dict__, + ) + + # Enum + validator = { + 'type': 'Enum', + 'version_from': 90300, + 'version_till': None, + 'possible_values': ('abc', 'def'), + } + ret = ValidatorFactory(validator) + self.assertIsInstance(ret, Enum) + self.assertEqual( + ret.__dict__, + Enum(version_from=validator['version_from'], version_till=validator['version_till'], + possible_values=validator['possible_values']).__dict__, + ) + + # EnumBool + validator = { + 'type': 'EnumBool', + 'version_from': 90300, + 'version_till': None, + 'possible_values': ('abc', 'def'), + } + ret = ValidatorFactory(validator) + self.assertIsInstance(ret, EnumBool) + self.assertEqual( + ret.__dict__, + EnumBool(version_from=validator['version_from'], version_till=validator['version_till'], + possible_values=validator['possible_values']).__dict__, + ) + + # String + validator = { + 'type': 'String', + 'version_from': 90300, + 'version_till': None, + } + ret = ValidatorFactory(validator) + self.assertIsInstance(ret, String) + self.assertEqual( + ret.__dict__, + String(version_from=validator['version_from'], version_till=validator['version_till']).__dict__, + ) + + def test__get_postgres_guc_validators(self): + # normal run + parameter = 'my_parameter' + + config = { + parameter: [{ + 'type': 'Bool', + 'version_from': 90300, + 'version_till': 90500, + }, { + 'type': 'EnumBool', + 'version_from': 90500, + 'version_till': 90600, + 'possible_values': [ + 'always', + ], + }] + } + ret = _get_postgres_guc_validators(config, parameter) + self.assertIsInstance(ret, tuple) + self.assertEqual(len(ret), 2) + self.assertIsInstance(ret[0], Bool) + self.assertIsInstance(ret[1], EnumBool) + + # log exceptions + del config[parameter][0]['type'] + + with patch('patroni.postgresql.validator.logger.warning') as mock_logger: + ret = _get_postgres_guc_validators(config, parameter) + self.assertIsInstance(ret, tuple) + self.assertEqual(len(ret), 1) + self.assertIsInstance(ret[0], EnumBool) + + mock_logger.assert_called_once() + mock_call = mock_logger.call_args[0] + self.assertEqual(mock_call[0], 'Faced an issue while parsing a validator for parameter `%s`: `%r`') + self.assertEqual(mock_call[1], parameter) + self.assertIsInstance(mock_call[2], ValidatorFactoryNoType) + + def test__read_postgres_gucs_validators_file(self): + # raise exception + with self.assertRaises(InvalidGucValidatorsFile) as exc: + _read_postgres_gucs_validators_file('random_file.yaml') + self.assertEqual( + str(exc.exception), + "Unexpected issue while reading parameters file `random_file.yaml`: `[Errno 2] No such file or directory: " + "'random_file.yaml'`." + ) + + def test__load_postgres_gucs_validators(self): + # log messages + with patch('os.walk', Mock(return_value=iter([('.', [], ['file.txt', 'random.yaml'])]))), \ + patch('patroni.postgresql.validator.logger.info') as mock_info, \ + patch('patroni.postgresql.validator.logger.warning') as mock_warning: + _load_postgres_gucs_validators() + mock_info.assert_called_once_with('Ignored a non-YAML file found under `available_parameters` directory: ' + '`%s`.', os.path.join('.', 'file.txt')) + mock_warning.assert_called_once() + self.assertIn( + "Unexpected issue while reading parameters file `{0}`: `[Errno 2] No such file or " + "directory:".format(os.path.join('.', 'random.yaml')), + mock_warning.call_args[0][0] + ) + + +@patch('subprocess.call', Mock(return_value=0)) +@patch('patroni.psycopg.connect', psycopg_connect) +class TestPostgresql2(BaseTestPostgresql): + + @patch('subprocess.call', Mock(return_value=0)) + @patch('os.rename', Mock()) + @patch('patroni.postgresql.CallbackExecutor', Mock()) + @patch.object(Postgresql, 'get_major_version', Mock(return_value=140000)) + @patch.object(Postgresql, 'is_running', Mock(return_value=True)) + def setUp(self): + super(TestPostgresql2, self).setUp() + + @patch('subprocess.check_output', Mock(return_value='\n'.join(mock_available_gucs.return_value).encode('utf-8'))) + def test_available_gucs(self): + gucs = self.p.available_gucs + self.assertIsInstance(gucs, CaseInsensitiveSet) + self.assertEqual(gucs, mock_available_gucs.return_value) diff --git a/tests/test_sync.py b/tests/test_sync.py index 0cfba7f8..78a500e3 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -7,11 +7,12 @@ from patroni.config import GlobalConfig from patroni.dcs import Cluster, SyncState from patroni.postgresql import Postgresql -from . import BaseTestPostgresql, psycopg_connect +from . import BaseTestPostgresql, psycopg_connect, mock_available_gucs @patch('subprocess.call', Mock(return_value=0)) @patch('patroni.psycopg.connect', psycopg_connect) +@patch.object(Postgresql, 'available_gucs', mock_available_gucs) class TestSync(BaseTestPostgresql): @patch('subprocess.call', Mock(return_value=0)) @@ -19,6 +20,7 @@ class TestSync(BaseTestPostgresql): @patch('patroni.postgresql.CallbackExecutor', Mock()) @patch.object(Postgresql, 'get_major_version', Mock(return_value=140000)) @patch.object(Postgresql, 'is_running', Mock(return_value=True)) + @patch.object(Postgresql, 'available_gucs', mock_available_gucs) def setUp(self): super(TestSync, self).setUp() self.p.config.write_postgresql_conf() From f3c80d57064adf53a7c010048dd9e523e43eec4e Mon Sep 17 00:00:00 2001 From: mikecaat <35882227+mikecaat@users.noreply.github.com> Date: Thu, 1 Jun 2023 04:22:30 +0900 Subject: [PATCH 02/22] Fix a minor error building a docker image for citus (#2705) This handles the following syntax error. $ docker build -t patroni-citus -f Dockerfile.citus . (snip) => ERROR [builder 2/3] RUN set -ex && export DEBIAN_FRONTEND=noninteractive && echo 0.5s - (snip) #5 0.456 /bin/sh: 1: Syntax error: end of file unexpected (expecting "fi") Co-authored-by: Masahiro Ikeda --- Dockerfile.citus | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.citus b/Dockerfile.citus index 2a10745e..36dcbb51 100644 --- a/Dockerfile.citus +++ b/Dockerfile.citus @@ -40,7 +40,7 @@ RUN set -ex \ echo "deb [signed-by=/etc/apt/trusted.gpg.d/citusdata_community.gpg] https://packagecloud.io/citusdata/community/debian/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/citusdata_community.list \ && curl -sL https://packagecloud.io/citusdata/community/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/citusdata_community.gpg \ && apt-get update -y \ - && apt-get -y install postgresql-$PG_MAJOR-citus-11.3 \ + && apt-get -y install postgresql-$PG_MAJOR-citus-11.3; \ fi \ && pip3 install dumb-init \ \ From af318b24730d4d37965682e1c9fbc97a4fd7660c Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 1 Jun 2023 07:28:29 -0400 Subject: [PATCH 03/22] Fix kubernetes behave tests (#2707) Starting from 1.27 there is containerd process, which also uses k3s binary and being detected by pidof. Therefore we will search for "k3s server" string in the process list instead of just "k3s". --- features/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/environment.py b/features/environment.py index c3fcce00..a4d22015 100644 --- a/features/environment.py +++ b/features/environment.py @@ -595,7 +595,7 @@ class KubernetesController(AbstractExternalDcsController): api_process = 'kube-apiserver' elif context.startswith('k3d-'): container = '{0}-server-0'.format(context) - api_process = 'k3s' + api_process = 'k3s server' else: return super(KubernetesController, self)._is_running() try: From 21e92fd166e8450655df99447e24d31dcd6b4b09 Mon Sep 17 00:00:00 2001 From: Polina Bungina <27892524+hughcapet@users.noreply.github.com> Date: Thu, 1 Jun 2023 14:06:11 +0200 Subject: [PATCH 04/22] Add env vars for custom bin names (#2706) --- docs/.DS_Store | Bin 0 -> 6148 bytes docs/ENVIRONMENT.rst | 7 +++++++ patroni/config.py | 5 +++++ tests/test_config.py | 3 ++- 4 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 docs/.DS_Store diff --git a/docs/.DS_Store b/docs/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..16d875f048b003f89438f610b00ecfc05d69253a GIT binary patch literal 6148 zcmeH~J&pn~427Thk&tL8DbsL(y+MTF1YBUnPJ=WO#fm;h=h<(?J6KYLuU74ZjM<6h0 J5P^Rs@C1L66M6su literal 0 HcmV?d00001 diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst index d2f8a49b..91816184 100644 --- a/docs/ENVIRONMENT.rst +++ b/docs/ENVIRONMENT.rst @@ -136,6 +136,13 @@ PostgreSQL - **PATRONI\_POSTGRESQL\_DATA\_DIR**: The location of the Postgres data directory, either existing or to be initialized by Patroni. - **PATRONI\_POSTGRESQL\_CONFIG\_DIR**: The location of the Postgres configuration directory, defaults to the data directory. Must be writable by Patroni. - **PATRONI\_POSTGRESQL\_BIN_DIR**: Path to PostgreSQL binaries. (pg_ctl, initdb, pg_controldata, pg_basebackup, postgres, pg_isready, pg_rewind) The default value is an empty string meaning that PATH environment variable will be used to find the executables. +- **PATRONI\_POSTGRESQL\_BIN\_PG\_CTL**: (optional) Custom name for ``pg_ctl`` binary. +- **PATRONI\_POSTGRESQL\_BIN\_INITDB**: (optional) Custom name for ``initdb`` binary. +- **PATRONI\_POSTGRESQL\_BIN\_PG\_CONTROLDATA**: (optional) Custom name for ``pg_controldata`` binary. +- **PATRONI\_POSTGRESQL\_BIN\_PG\_BASEBACKUP**: (optional) Custom name for ``pg_basebackup`` binary. +- **PATRONI\_POSTGRESQL\_BIN\_POSTGRES**: (optional) Custom name for ``postgres`` binary. +- **PATRONI\_POSTGRESQL\_BIN\_IS\_READY**: (optional) Custom name for ``pg_isready`` binary. +- **PATRONI\_POSTGRESQL\_BIN\_PG\_REWIND**: (optional) Custom name for ``pg_rewind`` binary. - **PATRONI\_POSTGRESQL\_PGPASS**: path to the `.pgpass `__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni. - **PATRONI\_REPLICATION\_USERNAME**: replication username; the user will be created during initialization. Replicas will use this user to access the replication source via streaming replication - **PATRONI\_REPLICATION\_PASSWORD**: replication password; the user will be created during initialization. diff --git a/patroni/config.py b/patroni/config.py index abd2371d..aa71d821 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -390,6 +390,11 @@ class Config(object): 'dir', 'file_size', 'file_num', 'loggers']) _set_section_values('raft', ['data_dir', 'self_addr', 'partner_addrs', 'password', 'bind_addr']) + for binary in ('pg_ctl', 'initdb', 'pg_controldata', 'pg_basebackup', 'postgres', 'pg_isready', 'pg_rewind'): + value = _popenv('POSTGRESQL_BIN_' + binary) + if value: + ret['postgresql'].setdefault('bin_name', {})[binary] = value + for first, second in (('restapi', 'allowlist_include_members'), ('ctl', 'insecure')): value = ret.get(first, {}).pop(second, None) if value: diff --git a/tests/test_config.py b/tests/test_config.py index 8223a62f..57a717f5 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -70,7 +70,8 @@ class TestConfig(unittest.TestCase): 'PATRONI_REPLICATION_USERNAME': 'replicator', 'PATRONI_REPLICATION_PASSWORD': 'rep-pass', 'PATRONI_admin_PASSWORD': 'admin', - 'PATRONI_admin_OPTIONS': 'createrole,createdb' + 'PATRONI_admin_OPTIONS': 'createrole,createdb', + 'PATRONI_POSTGRESQL_BIN_POSTGRES': 'sergtsop' }) config = Config('postgres0.yml') with patch.object(Config, '_load_config_file', Mock(return_value={'restapi': {}})): From 4b960477bb8ae436cbc49c45cf3e6bb360f26a87 Mon Sep 17 00:00:00 2001 From: Israel Date: Tue, 6 Jun 2023 03:21:59 -0300 Subject: [PATCH 05/22] Add docstrings to `patroni.ctl` (#2687) References: PAT-90. --- patroni/ctl.py | 932 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 872 insertions(+), 60 deletions(-) diff --git a/patroni/ctl.py b/patroni/ctl.py index 612a97db..8e231466 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -1,6 +1,16 @@ -''' -Patroni Control -''' +"""Implement ``patronictl``: a command-line application which utilises the REST API to perform cluster operations. + +:var CONFIG_DIR_PATH: path to Patroni configuration directory as per :func:`click.get_app_dir` output. +:var CONFIG_FILE_PATH: default path to ``patronictl.yaml`` configuration file. +:var DCS_DEFAULTS: auxiliary dictionary to build the DCS section of the configuration file. Mainly used to help parsing + ``--dcs-url`` command-line option of ``patronictl``. + +.. note:: + Most of the ``patronictl`` commands (``restart``/``reinit``/``pause``/``resume``/``show-config``/``edit-config`` and + similar) require the ``group`` argument and work only for that specific Citus ``group``. + If not specified in the command line the ``group`` might be taken from the configuration file. + If it is also missing in the configuration file we assume that this is just a normal Patroni cluster (not Citus). +""" import click import codecs @@ -54,22 +64,59 @@ DCS_DEFAULTS: Dict[str, Dict[str, Any]] = { class PatroniCtlException(click.ClickException): + """Raised upon issues faced by ``patronictl`` utility.""" + pass class PatronictlPrettyTable(PrettyTable): + """Utilitary class to print pretty tables. + + Extend :class:`~prettytable.PrettyTable` to make it print custom information in the header line. The idea is to + print a header line like this: + + ``` + + Cluster: batman --------+--------+---------+----+-----------+ + ``` + + Instead of the default header line which would contain only dash and plus characters. + """ def __init__(self, header: str, *args: Any, **kwargs: Any) -> None: + """Create a :class:`PatronictlPrettyTable` instance with the given *header*. + + :param header: custom string to be put in the first header line of the table. + :param args: positional arguments to be passed to :class:`~prettytable.PrettyTable` constructor. + :param kwargs: keyword arguments to be passed to :class:`~prettytable.PrettyTable` constructor. + """ super(PatronictlPrettyTable, self).__init__(*args, **kwargs) self.__table_header = header self.__hline_num = 0 self.__hline: str def __build_header(self, line: str) -> str: + """Build the custom header line for the table. + + .. note:: + Expected to be called only against the very first header line of the table. + + :param line: the original header line. + + :returns: the modified header line. + """ header = self.__table_header[:len(line) - 2] return "".join([line[0], header, line[1 + len(header):]]) def _stringify_hrule(self, *args: Any, **kwargs: Any) -> str: + """Get the string representation of a header line. + + Inject the custom header line, if processing the first header line. + + .. note:: + New implementation for injecting a custom header line, which is used from :mod:`prettytable` 2.2.0 onwards. + + :returns: string representation of a header line. + """ ret = super(PatronictlPrettyTable, self)._stringify_hrule(*args, **kwargs) where = args[1] if len(args) > 1 else kwargs.get('where') if where == 'top_' and self.__table_header: @@ -78,12 +125,30 @@ class PatronictlPrettyTable(PrettyTable): return ret def _is_first_hline(self) -> bool: + """Check if the current line being processed is the very first line of the header. + + :returns: ``True`` if processing the first header line, ``False`` otherwise. + """ return self.__hline_num == 0 def _set_hline(self, value: str) -> None: + """Set header line string representation. + + :param value: string representing a header line. + """ self.__hline = value def _get_hline(self) -> str: + """Get string representation of a header line. + + Inject the custom header line, if processing the first header line. + + .. note:: + Original implementation for injecting a custom header line, and is used up to :mod:`prettytable` 2.2.0. From + :mod:`prettytable` 2.2.0 onwards :func:`_stringify_hrule` is used instead. + + :returns: string representing a header line. + """ ret = self.__hline # Inject nice table header @@ -99,20 +164,22 @@ class PatronictlPrettyTable(PrettyTable): def parse_dcs(dcs: Optional[str]) -> Optional[Dict[str, Any]]: """Parse a DCS URL. - :param dcs: the DCS URL in the format ``DCS://HOST:PORT``. ``DCS`` can be one among + :param dcs: the DCS URL in the format ``DCS://HOST:PORT``. ``DCS`` can be one among: + * ``consul`` * ``etcd`` * ``etcd3`` * ``exhibitor`` * ``zookeeper`` - If ``DCS`` is not specified, it assumes ``etcd`` by default. If ``HOST`` is not specified, it assumes - ``localhost`` by default. If ``PORT`` is not specified, it assumes the default port of the given ``DCS``. + If ``DCS`` is not specified, assume ``etcd`` by default. If ``HOST`` is not specified, assume ``localhost`` by + default. If ``PORT`` is not specified, assume the default port of the given ``DCS``. :returns: ``None`` if *dcs* is ``None``, otherwise a dictionary. The dictionary represents *dcs* as if it were parsed from the Patroni configuration file. - :raises PatroniCtlException: if the DCS name in *dcs* is not valid. + :raises: + :class:`PatroniCtlException`: if the DCS name in *dcs* is not valid. :Example: @@ -147,6 +214,17 @@ def parse_dcs(dcs: Optional[str]) -> Optional[Dict[str, Any]]: def load_config(path: str, dcs_url: Optional[str]) -> Dict[str, Any]: + """Load configuration file from *path* and optionally override its DCS configuration with *dcs_url*. + + :param path: path to the configuration file. + :param dcs_url: the DCS URL in the format ``DCS://HOST:PORT``, e.g. ``etcd3://random.com:2399``. If given override + whatever DCS is set in the configuration file. + + :returns: a dictionary representing the configuration. + + :raises: + :class:`PatroniCtlException`: if *path* does not exist or is not readable. + """ from patroni.config import Config if not (os.path.exists(path) and os.access(path, os.R_OK)): @@ -187,17 +265,49 @@ role_choice = click.Choice(['leader', 'primary', 'standby-leader', 'replica', 's @option_insecure @click.pass_context def ctl(ctx: click.Context, config_file: str, dcs_url: Optional[str], insecure: bool) -> None: + """Entry point of ``patronictl`` utility. + + Load the configuration file. + + .. note:: + Besides *dcs_url* and *insecure*, which are used to override DCS configuration section and ``ctl.insecure`` + setting, you can also override the value of ``log.level``, by default ``WARNING``, through either of these + environemnt variables: + * ``LOGLEVEL`` + * ``PATRONI_LOGLEVEL`` + * ``PATRONI_LOG_LEVEL`` + + :param ctx: click context to be passed to sub-commands. + :param config_file: path to the configuration file. + :param dcs_url: the DCS URL in the format ``DCS://HOST:PORT``, e.g. ``etcd3://random.com:2399``. If given override + whatever DCS is set in the configuration file. + :param insecure: if ``True`` allow SSL connections without client certiticates. Override what is configured through + ``ctl.insecure` in the configuration file. + """ level = 'WARNING' for name in ('LOGLEVEL', 'PATRONI_LOGLEVEL', 'PATRONI_LOG_LEVEL'): level = os.environ.get(name, level) logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=level) logging.captureWarnings(True) # Capture eventual SSL warning ctx.obj = load_config(config_file, dcs_url) - # backward compatibility for configuration file where ctl section is not define + # backward compatibility for configuration file where ctl section is not defined ctx.obj.setdefault('ctl', {})['insecure'] = ctx.obj.get('ctl', {}).get('insecure') or insecure def get_dcs(config: Dict[str, Any], scope: str, group: Optional[int]) -> AbstractDCS: + """Get the DCS object. + + :param config: Patroni configuration. + :param scope: cluster name. + :param group: if *group* is defined, use it to select which alternative Citus group this DCS refers to. If *group* + is ``None`` and a Citus configuration exists, assume this is the coordinator. Coordinator has the group ``0``. + Refer to the module note for more details. + + :returns: a subclass of :class:`~patroni.dcs.AbstractDCS`, according to the DCS technology that is configured. + + :raises: + :class:`PatroniCtlException`: if not suitable DCS configuration could be found. + """ config.update({'scope': scope, 'patronictl': True}) if group is not None: config['citus'] = {'group': group} @@ -213,6 +323,15 @@ def get_dcs(config: Dict[str, Any], scope: str, group: Optional[int]) -> Abstrac def request_patroni(member: Member, method: str = 'GET', endpoint: Optional[str] = None, data: Optional[Any] = None) -> urllib3.response.HTTPResponse: + """Perform a request to Patroni REST API. + + :param member: DCS member, used to get the base URL of its REST API server. + :param method: HTTP method to be used, e.g. ``GET``. + :param endpoint: URL path of the request, e.g. ``patroni``. + :param data: anything to be used as the request body. + + :returns: the response for the request. + """ ctx = click.get_current_context() # the current click context request_executor = ctx.obj.get('__request_patroni') if not request_executor: @@ -222,6 +341,30 @@ def request_patroni(member: Member, method: str = 'GET', def print_output(columns: Optional[List[str]], rows: List[List[Any]], alignment: Optional[Dict[str, str]] = None, fmt: str = 'pretty', header: str = '', delimiter: str = '\t') -> None: + """Print tabular information. + + :param columns: list of column names. + :param rows: list of rows. Each item is a list of values for the columns. + :param alignment: alignment to be applied to column values. Each key is the name of a column to be aligned, and the + corresponding value can be one among: + + * ``l``: left-aligned + * ``c``: center-aligned + * ``r``: right-aligned + + A key in the dictionary is only required for a column that needs a specific alignment. Only apply when *fmt* is + either ``pretty`` or ``topology``. + :param fmt: the printing format. Can be one among: + + * ``json``: to print as a JSON string -- array of objects; + * ``yaml`` or ``yml``: to print as a YAML string; + * ``tsv``: to print a table of separated values, by default by tab; + * ``pretty``: to print a pretty table; + * ``topology``: similar to *pretty*, but with a topology view when printing cluster members. + :param header: a string to be included in the first line of the table header, typically the cluster name. Only + apply when *fmt* is either ``pretty`` or ``topology``. + :param delimiter: the character to be used as delimiter when *fmt* is ``tsv``. + """ if fmt in {'json', 'yaml', 'yml'}: elements = [{k: v for k, v in zip(columns or [], r) if not header or str(v)} for r in rows] func = json.dumps if fmt == 'json' else format_config_for_editing @@ -232,16 +375,22 @@ def print_output(columns: Optional[List[str]], rows: List[List[Any]], alignment: i = columns.index('Tags') for row in rows: if row[i]: + # Member tags are printed in YAML block format if *fmt* is ``pretty``. If *fmt* is either ``tsv`` + # or ``topology``, then write in the YAML flow format, which is similar to JSON row[i] = format_config_for_editing(row[i], fmt != 'pretty').strip() if list_cluster and header and fmt != 'tsv': # skip cluster name and maybe Citus group if pretty-printing skip_cols = 2 if ' (group: ' in header else 1 columns = columns[skip_cols:] if columns else [] rows = [row[skip_cols:] for row in rows] + # In ``tsv`` format print cluster name in every row as the first column if fmt == 'tsv': for r in ([columns] if columns else []) + rows: click.echo(delimiter.join(map(str, r))) + # In ``pretty`` and ``topology`` formats print the cluster name only once, in the very first header line else: + # If any value is multi-line, then add horizontal between all table rows while printing to get a clear + # visual separation of rows. hrules = ALL if any(any(isinstance(c, str) and '\n' in c for c in r) for r in rows) else FRAME table = PatronictlPrettyTable(header, columns, hrules=hrules) table.align = 'l' @@ -253,15 +402,26 @@ def print_output(columns: Optional[List[str]], rows: List[List[Any]], alignment: def watching(w: bool, watch: Optional[int], max_count: Optional[int] = None, clear: bool = True) -> Iterator[int]: - """ - >>> len(list(watching(True, 1, 0))) - 1 - >>> len(list(watching(True, 1, 1))) - 2 - >>> len(list(watching(True, None, 0))) - 1 - """ + """Yield a value every ``x`` seconds. + Used to run a command with a watch-based aproach. + + :param w: if ``True`` and *watch* is ``None``, then *watch* assumes the value ``2``. + :param watch: amount of seconds to wait before yielding another value. + :param max_count: maximum number of yielded values. If ``None`` keep yielding values indefinitely. + :param clear: if the screen should be cleared out at each iteration. + + :yields: ``0`` each time *watch* seconds have passed. + + :Example: + + >>> len(list(watching(True, 1, 0))) + 1 + >>> len(list(watching(True, 1, 1))) + 2 + >>> len(list(watching(True, None, 0))) + 1 + """ if w and not watch: watch = 2 if watch and clear: @@ -282,10 +442,28 @@ def watching(w: bool, watch: Optional[int], max_count: Optional[int] = None, cle def get_all_members(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], role: str = 'leader') -> Iterator[Member]: + """Get all cluster members that have the given *role*. + + :param obj: the Patroni configuration. + :param cluster: the Patroni cluster. + :param group: filter which Citus group we should get members from. If ``None`` get from all groups. + :param role: role to filter members. Can be one among: + + * ``primary`` or ``master``: the primary PostgreSQL instance; + * ``replica`` or ``standby``: a standby PostgreSQL instance; + * ``leader``: the leader of a Patroni cluster. Can also be used to get the leader of a Patroni standby cluster; + * ``standby-leader``: the leader of a Patroni standby cluster; + * ``any``: matches any node independent of its role. + + :yields: members that have the given *role*. + """ clusters = {0: cluster} if obj.get('citus') and group is None: clusters.update(cluster.workers) if role in ('leader', 'master', 'primary', 'standby-leader'): + # In the DCS the members' role can be one among: ``primary``, ``master``, ``replica`` or ``standby_leader``. + # ``primary`` and ``master`` are the same thing, so we map both to ``master`` to have a simpler ``if``. + # In a future release we might remove ``master`` from the available roles for the DCS members. role = {'primary': 'master', 'standby-leader': 'standby_leader'}.get(role, role) for cluster in clusters.values(): if cluster.leader is not None and cluster.leader.name and\ @@ -303,13 +481,40 @@ def get_all_members(obj: Dict[str, Any], cluster: Cluster, def get_any_member(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], - role: str = 'leader', member: Optional[str] = None) -> Optional[Member]: + role: Optional[str] = None, member: Optional[str] = None) -> Optional[Member]: + """Get the first found cluster member that has the given *role*. + + :param obj: the Patroni configuration. + :param cluster: the Patroni cluster. + :param group: filter which Citus group we should get members from. If ``None`` get from all groups. + :param role: role to filter members. See :func:`get_all_members` for available options. + :param member: if specified, then besides having the given *role*, the cluster member's name should be *member*. + + :returns: the first found cluster member that has the given *role*. + + :raises: + :class:`PatroniCtlException`: if both *role* and *member* are provided. + """ + if member is not None: + if role is not None: + raise PatroniCtlException('--role and --member are mutually exclusive options') + role = 'any' + elif role is None: + role = 'leader' + for m in get_all_members(obj, cluster, group, role): if member is None or m.name == member: return m def get_all_members_leader_first(cluster: Cluster) -> Iterator[Member]: + """Get all cluster members, with the cluster leader being yielded first. + + .. note:: + Only yield members that have a ``restapi.connect_address`` configured. + + :yields: all cluster members, with the leader first. + """ leader_name = cluster.leader.member.name if cluster.leader and cluster.leader.member.api_url else None if leader_name and cluster.leader: yield cluster.leader.member @@ -319,7 +524,29 @@ def get_all_members_leader_first(cluster: Cluster) -> Iterator[Member]: def get_cursor(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], connect_parameters: Dict[str, Any], - role: str = 'leader', member_name: Optional[str] = None) -> Union['cursor', 'Cursor[Any]', None]: + role: Optional[str] = None, member_name: Optional[str] = None) -> Union['cursor', 'Cursor[Any]', None]: + """Get a cursor object to execute queries against a member that has the given *role* or *member_name*. + + .. note:: + Besides what is passed through *connect_parameters*, this function also sets the following parameters: + * ``fallback_application_name``: as ``Patroni ctl``; + * ``connect_timeout``: as ``5``. + + :param obj: the Patroni configuration. + :param cluster: the Patroni cluster. + :param group: filter which Citus group we should get members to create a cursor against. If ``None`` consider + members from all groups. + :param connect_parameters: database connection parameters. + :param role: role to filter members. See :func:`get_all_members` for available options. + :param member_name: if specified, then besides having the given *role*, the cluster member's name should be + *member_name*. + + :returns: a cursor object to execute queries against the database. Can be either: + + * A :class:`psycopg.Cursor` if using :mod:`psycopg`; or + * A :class:`psycopg2.extensions.cursor` if using :mod:`psycopg2`; + * ``None`` if not able to get a cursor that attendees *role* and *member_name*. + """ member = get_any_member(obj, cluster, group, role=role, member=member_name) if member is None: return None @@ -334,9 +561,13 @@ def get_cursor(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], conn from . import psycopg conn = psycopg.connect(**params) cursor = conn.cursor() + # If we want ``any`` node we are fine to return the cursor + # If we want the Patroni leader node, :func:`get_any_member` already checks that for us if role in ('any', 'leader'): return cursor + # If we want something other than ``any`` or ``leader``, then we do not rely only on the DCS information about + # members, but rather double check the underlying Postgres status. cursor.execute('SELECT pg_catalog.pg_is_in_recovery()') row = cursor.fetchone() in_recovery = not row or row[0] @@ -352,6 +583,57 @@ def get_cursor(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], conn def get_members(obj: Dict[str, Any], cluster: Cluster, cluster_name: str, member_names: List[str], role: str, force: bool, action: str, ask_confirmation: bool = True, group: Optional[int] = None) -> List[Member]: + """Get the list of members based on the given filters. + + .. note:: + Contain some filtering and checks processing that are common to several actions that are exposed + by `patronictl`, like: + + * Get members of *cluster* that respect the given *member_names*, *role*, and *group*; + * Bypass confirmations; + * Prompt user for information that has not been passed through the command-line options; + * etc. + + Designed to handle both attended and unattended ``patronictl`` commands execution that need to retrieve and + validate the members before doing anything. + + In the very end may call :func:`confirm_members_action` to ask if the user would like to proceed with *action* + over the retrieved members. That won't actually perform the action, but it works as the "last confirmation" + before the *action* is processed by the caller method. + + Additional checks can also be implemented in the caller method, in which case you might want to pass + ``ask_confirmation=False``, and later call :func:`confirm_members_action` manually in the caller method. That + way the workflow won't look broken to the user that is interacting with ``patronictl``. + + :param obj: Patroni configuration. + :param cluster: Patroni cluster. + :param cluster_name: name of the Patroni cluster. + :param member_names: used to filter which members should take the *action* based on their names. Each item is the + name of a Patroni member, as per ``name`` configuration. If *member_names* is an empty :class:`tuple` no filters + are applied based on names. + :param role: used to filter which members should take the *action* based on their role. See :func:`get_all_members` + for available options. + :param force: if ``True``, then it won't ask for confirmations at any point nor prompt the user to select values + for options that were not specified through the command-line. + :param action: the action that is being processed, one among: + + * ``reload``: reload PostgreSQL configuration; or + * ``restart``: restart PostgreSQL; or + * ``reinitialize``: reinitialize PostgreSQL data directory; or + * ``flush``: discard scheduled actions. + :param ask_confirmation: if ``False``, then it won't ask for the final confirmation regarding the *action* before + returning the list of members. Usually useful as ``False`` if you want to perform additional checks in + the caller method besides the checks that are performed through this generic method. + :param group: filter which Citus group we should get members from. If ``None`` consider members from all groups. + + :returns: a list of members that respect the given filters. + + :raises: + :class:`PatroniCtlException`: if + * Cluster does not have members that match the given *role*; or + * Cluster does not have members that match the given *member_names*; or + * No member with given *role* is found among the specified *member_names*. + """ members = list(get_all_members(obj, cluster, group, role)) candidates = {m.name for m in members} @@ -383,6 +665,22 @@ def get_members(obj: Dict[str, Any], cluster: Cluster, cluster_name: str, member def confirm_members_action(members: List[Member], force: bool, action: str, scheduled_at: Optional[datetime.datetime] = None) -> None: + """Ask for confirmation if *action* should be taken by *members*. + + :param members: list of member which will take the *action*. + :param force: if ``True`` skip the confirmation prompt and allow the *action* to proceed. + :param action: the action that is being processed, one among: + + * ``reload``: reload PostgreSQL configuration; or + * ``restart``: restart PostgreSQL; or + * ``reinitialize``: reinitialize PostgreSQL data directory; or + * ``flush``: discard scheduled actions. + :param scheduled_at: timestamp at which the *action* should be scheduled to. If ``None`` *action* is taken + immediately. + + :raises: + :class:`PatroniCtlException`: if the user aborted the *action*. + """ if scheduled_at: if not force: confirm = click.confirm('Are you sure you want to schedule {0} of members {1} at {2}?' @@ -405,13 +703,26 @@ def confirm_members_action(members: List[Member], force: bool, action: str, @click.pass_obj def dsn(obj: Dict[str, Any], cluster_name: str, group: Optional[int], role: Optional[str], member: Optional[str]) -> None: - if member is not None: - if role is not None: - raise PatroniCtlException('--role and --member are mutually exclusive options') - role = 'any' - elif role is None: - role = 'leader' + """Process ``dsn`` command of ``patronictl`` utility. + Get DSN to connect to *member*. + + .. note:: + If no *role* nor *member* is given assume *role* as ``leader``. + + :param obj: Patroni configuration. + :param cluster_name: name of the Patroni cluster. + :param group: filter which Citus group we should get members to get DSN from. Refer to the module note for more + details. + :param role: filter which members to get DSN from based on their role. See :func:`get_all_members` for available + options. + :param member: filter which member to get DSN from based on its name. + + :raises: + :class:`PatroniCtlException`: if + * both *role* and *member* are provided; or + * No member matches requested *member* or *role*. + """ cluster = get_dcs(obj, cluster_name, group).get_cluster() m = get_any_member(obj, cluster, group, role=role, member=member) if m is None: @@ -452,13 +763,33 @@ def query( dbname: Optional[str], fmt: str = 'tsv' ) -> None: - if member is not None: - if role is not None: - raise PatroniCtlException('--role and --member are mutually exclusive options') - role = 'any' - elif role is None: - role = 'leader' + """Process ``query`` command of ``patronictl`` utility. + Perform a Postgres query in a Patroni node. + + :param obj: Patroni configuration. + :param cluster_name: name of the Patroni cluster. + :param group: filter which Citus group we should get members from to perform the query. Refer to the module note for + more details. + :param role: filter which members to perform the query against based on their role. See :func:`get_all_members` for + available options. + :param member: filter which member to perform the query against based on its name. + :param w: perform query with watch-based approach every 2 seconds. + :param watch: perform query with watch-based approach every *watch* seconds. + :param delimiter: column delimiter when *fmt* is ``tsv``. + :param command: SQL query to execute. + :param p_file: path to file containing SQL query to execute. + :param password: if ``True`` then prompt for password. + :param username: name of the database user. + :param dbname: name of the database. + :param fmt: the output table printing format. See :func:`print_output` for available options. + + :raises: + :class:`PatroniCtlException`: if: + * if * both *role* and *member* are provided; or + * both *file* and *command* are provided; or + * neither *file* nor *command* is provided. + """ if p_file is not None: if command is not None: raise PatroniCtlException('--file and --command are mutually exclusive options') @@ -489,8 +820,36 @@ def query( def query_member(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], - cursor: Union['cursor', 'Cursor[Any]', None], member: Optional[str], role: str, + cursor: Union['cursor', 'Cursor[Any]', None], member: Optional[str], role: Optional[str], command: str, connect_parameters: Dict[str, Any]) -> Tuple[List[List[Any]], Optional[List[Any]]]: + """Execute SQL *command* against a member. + + :param obj: Patroni configuration. + :param cluster: the Patroni cluster. + :param group: filter which Citus group we should get members from to perform the query. Refer to the module note for + more details. + :param cursor: cursor through which *command* is executed. If ``None`` a new cursor is instantiated through + :func:`get_cursor`. + :param member: filter which member to create a cursor against based on its name, if *cursor* is ``None``. + :param role: filter which member to create a cursor against based on their role, if *cursor* is ``None``. See + :func:`get_all_members` for available options. + :param command: SQL command to be executed. + :param connect_parameters: connection parameters to be passed down to :func:`get_cursor`, if *cursor* is ``None``. + + :returns: a tuple composed of two items: + + * List of rows returned by the executed *command*; + * List of columns related to the rows returned by the executed *command*. + + If an error occurs while executing *command*, then returns the following values in the tuple: + + * List with 2 items: + + * Current timestamp; + * Error message. + + * ``None``. + """ from . import psycopg try: if cursor is None: @@ -521,6 +880,24 @@ def query_member(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], @option_format @click.pass_obj def remove(obj: Dict[str, Any], cluster_name: str, group: Optional[int], fmt: str) -> None: + """Process ``remove`` command of ``patronictl`` utility. + + Remove cluster *cluster_name* from the DCS. + + :param obj: Patroni configuration. + :param cluster_name: name of the cluster which information will be wiped out of the DCS. + :param group: which Citus group should have its information wiped out of the DCS. Refer to the module note for more + details. + :param fmt: the output table printing format. See :func:`print_output` for available options. + + :raises: + :class:`PatroniCtlException`: if: + * Patroni is running on a Citus cluster, but no *group* was specified; or + * *cluster_name* does not exist; or + * user did not type the expected confirmation message when prompted for confirmation; or + * use did not type the correct leader name when requesting removal of a healthy cluster. + + """ dcs = get_dcs(obj, cluster_name, group) cluster = dcs.get_cluster() @@ -549,6 +926,15 @@ def remove(obj: Dict[str, Any], cluster_name: str, group: Optional[int], fmt: st def check_response(response: urllib3.response.HTTPResponse, member_name: str, action_name: str, silent_success: bool = False) -> bool: + """Check an HTTP response and print a status message. + + :param response: the response to be checked. + :param member_name: name of the member associated with the *response*. + :param action_name: action associated with the *response*. + :param silent_success: if a status message should be skipped upon a successful *response*. + + :returns: ``True`` if the response indicates a sucessful operation (HTTP status < ``400``), ``False`` otherwise. + """ if response.status >= 400: click.echo('Failed: {0} for member {1}, status code={2}, ({3})'.format( action_name, member_name, response.status, response.data.decode('utf-8') @@ -560,6 +946,29 @@ def check_response(response: urllib3.response.HTTPResponse, member_name: str, def parse_scheduled(scheduled: Optional[str]) -> Optional[datetime.datetime]: + """Parse a string *scheduled* timestamp as a :class:`~datetime.datetime` object. + + :param scheduled: string representation of the timestamp. May also be ``now``. + + :returns: the corresponding :class:`~datetime.datetime` object, if *scheduled* is not ``now``, otherwise ``None``. + + :raises: + :class:`PatroniCtlException`: if unable to parse *scheduled* from :class:`str` to :class:`~datetime.datetime`. + + :Example: + + >>> parse_scheduled(None) is None + True + + >>> parse_scheduled('now') is None + True + + >>> parse_scheduled('2023-05-29T04:32:31') + datetime.datetime(2023, 5, 29, 4, 32, 31, tzinfo=tzlocal()) + + >>> parse_scheduled('2023-05-29T04:32:31-3') + datetime.datetime(2023, 5, 29, 4, 32, 31, tzinfo=tzoffset(None, -10800)) + """ if scheduled is not None and (scheduled or 'now') != 'now': try: scheduled_at = dateutil.parser.parse(scheduled) @@ -582,6 +991,17 @@ def parse_scheduled(scheduled: Optional[str]) -> Optional[datetime.datetime]: @click.pass_obj def reload(obj: Dict[str, Any], cluster_name: str, member_names: List[str], group: Optional[int], force: bool, role: str) -> None: + """Process ``reload`` command of ``patronictl`` utility. + + Reload configuration of cluster members based on given filters. + + :param obj: Patroni configuration. + :param cluster_name: name of the Patroni cluster. + :param member_names: name of the members which configuration should be reloaded. + :param group: filter which Citus group we should reload members. Refer to the module note for more details. + :param force: perform the reload without asking for confirmations. + :param role: role to filter members. See :func:`get_all_members` for available options. + """ dcs = get_dcs(obj, cluster_name, group) cluster = dcs.get_cluster() @@ -620,6 +1040,28 @@ def reload(obj: Dict[str, Any], cluster_name: str, member_names: List[str], def restart(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member_names: List[str], force: bool, role: str, p_any: bool, scheduled: Optional[str], version: Optional[str], pending: bool, timeout: Optional[str]) -> None: + """Process ``restart`` command of ``patronictl`` utility. + + Restart Postgres on cluster members based on given filters. + + :param obj: Patroni configuration. + :param cluster_name: name of the Patroni cluster. + :param group: filter which Citus group we should restart members. Refer to the module note for more details. + :param member_names: name of the members that should be restarted. + :param force: perform the restart without asking for confirmations. + :param role: role to filter members. See :func:`get_all_members` for available options. + :param p_any: restart a single and random member among the ones that match the given filters. + :param scheduled: timestamp when the restart should be scheduled to occur. If ``now`` restart immediately. + :param version: restart only members which Postgres version is less than *version*. + :param pending: restart only members that are flagged as ``pending restart``. + :param timeout: timeout for the restart operation. If timeout is reached a failover may occur in the cluster. + + :raises: + :class:`PatroniCtlException`: if: + * *scheduled* could not be parsed; or + * *version* could not be parsed; or + * a restart is attempted against a cluster that is in maintenance mode. + """ cluster = get_dcs(obj, cluster_name, group).get_cluster() members = get_members(obj, cluster, cluster_name, member_names, role, force, 'restart', False, group=group) @@ -688,6 +1130,20 @@ def restart(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member @click.pass_obj def reinit(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member_names: List[str], force: bool, wait: bool) -> None: + """Process ``reinit`` command of ``patronictl`` utility. + + Reinitialize cluster members based on given filters. + + .. note:: + Only reinitialize replica members, not a leader. + + :param obj: Patroni configuration. + :param cluster_name: name of the Patroni cluster. + :param group: filter which Citus group we should reinit members. Refer to the module note for more details. + :param member_names: name of the members that should be reinitialized. + :param force: perform the restart without asking for confirmations. + :param wait: wait for the operation to complete. + """ cluster = get_dcs(obj, cluster_name, group).get_cluster() members = get_members(obj, cluster, cluster_name, member_names, 'replica', force, 'reinitialize', group=group) @@ -723,13 +1179,36 @@ def reinit(obj: Dict[str, Any], cluster_name: str, group: Optional[int], def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: str, group: Optional[int], leader: Optional[str], candidate: Optional[str], force: bool, scheduled: Optional[str] = None) -> None: - """ - We want to trigger a failover or switchover for the specified cluster name. + """Perform a failover or a switchover operation in the cluster. - We verify that the cluster name, leader name and candidate name are correct. - If so, we trigger an action and keep the client up to date. - """ + Informational messages are printed in the console during the operation, as well as the list of members before and + after the operation, so the user can follow the operation status. + .. note:: + If not able to perform the operation through the REST API, write directly to the DCS as a fall back. + + :param obj: Patroni configuration. + :param action: action to be taken -- ``failover`` or ``switchover``. + :param cluster_name: name of the Patroni cluster. + :param group: filter Citus group within we should perform a failover or switchover. If ``None``, user will be + prompted for filling it -- unless *force* is ``True``, in which case an exception is raised. + :param leader: name of the current leader member. + :param candidate: name of a standby member to be promoted. Nodes that are tagged with ``nofailover`` cannot be used. + :param force: perform the failover or switchover without asking for confirmations. + :param scheduled: timestamp when the switchover should be scheduled to occur. If ``now`` perform immediately. + + :raises: + :class:`PatroniCtlException`: if: + * Patroni is running on a Citus cluster, but no *group* was specified; or + * a switchover was requested by the cluster has no leader; or + * *leader* does not match the current leader of the cluster; or + * cluster has no candidates available for the operation; or + * no *candidate* is given for a failover operation; or + * *leader* and *candidate* are the same; or + * *candidate* is not a member of the cluster; or + * trying to schedule a switchover in a cluster that is in maintenance mode; or + * user aborts the operation. + """ dcs = get_dcs(obj, cluster_name, group) cluster = dcs.get_cluster() click.echo('Current cluster topology') @@ -846,6 +1325,25 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s @click.pass_obj def failover(obj: Dict[str, Any], cluster_name: str, group: Optional[int], leader: Optional[str], candidate: Optional[str], force: bool) -> None: + """Process ``failover`` command of ``patronictl`` utility. + + Perform a failover operation immediately in the cluster. + + .. note:: + If *leader* is given perform a switchover instead of a failover. + + .. seealso:: + Refer to :func:`_do_failover_or_switchover` for details. + + :param obj: Patroni configuration. + :param cluster_name: name of the Patroni cluster. + :param group: filter Citus group within we should perform a failover or switchover. If ``None``, user will be + prompted for filling it -- unless *force* is ``True``, in which case an exception is raised by + :func:`_do_failover_or_switchover`. + :param leader: name of the current leader member. + :param candidate: name of a standby member to be promoted. Nodes that are tagged with ``nofailover`` cannot be used. + :param force: perform the failover or switchover without asking for confirmations. + """ action = 'switchover' if leader else 'failover' _do_failover_or_switchover(obj, action, cluster_name, group, leader, candidate, force) @@ -861,11 +1359,60 @@ def failover(obj: Dict[str, Any], cluster_name: str, group: Optional[int], @click.pass_obj def switchover(obj: Dict[str, Any], cluster_name: str, group: Optional[int], leader: Optional[str], candidate: Optional[str], force: bool, scheduled: Optional[str]) -> None: + """Process ``switchover`` command of ``patronictl`` utility. + + Perform a switchover operation in the cluster. + + .. seealso:: + Refer to :func:`_do_failover_or_switchover` for details. + + :param obj: Patroni configuration. + :param cluster_name: name of the Patroni cluster. + :param group: filter Citus group within we should perform a switchover. If ``None``, user will be prompted for + filling it -- unless *force* is ``True``, in which case an exception is raised by + :func:`_do_failover_or_switchover`. + :param leader: name of the current leader member. + :param candidate: name of a standby member to be promoted. Nodes that are tagged with ``nofailover`` cannot be used. + :param force: perform the switchover without asking for confirmations. + :param scheduled: timestamp when the switchover should be scheduled to occur. If ``now`` perform immediately. + """ _do_failover_or_switchover(obj, 'switchover', cluster_name, group, leader, candidate, force, scheduled) def generate_topology(level: int, member: Dict[str, Any], topology: Dict[str, List[Dict[str, Any]]]) -> Iterator[Dict[str, Any]]: + """Recursively yield members with their names adjusted according to their *level* in the cluster topology. + + .. note:: + The idea is to get a tree view of the members when printing their names. For example, suppose you have a + cascading replication composed of 3 nodes, say ``postgresql0``, ``postgresql1``, and ``postgresql2``. This + function would adjust their names to be like this: + + * ``'postgresql0'`` -> ``'postgresql0'`` + * ``'postgresql1'`` -> ``'+ postgresql1'`` + * ``'postgresql2'`` -> ``' + postgresql2'`` + + So, if you ever print their names line by line, you would see something like this: + + .. code-block:: + + postgresql0 + + postgresql1 + + postgresql2 + + :param level: the current level being inspected in the *topology*. + :param member: information about the current member being inspected in *level* of *topology*. Should countain at + least this key: + * ``name``: name of the node, according to ``name`` configuration; + + But may contain others, which although ignored by this function, will be yielded as part of the resulting + object. The value of key ``name`` is changed as explained in the note. + + :param topology: each key is the name of a node which has at least one replica attached to it. The corresponding + value is a list of the attached replicas, each of them with the same structure described for *member*. + + :yields: the current member with its name changed. Besides that reyield values from recursive calls. + """ members = topology.get(member['name'], []) if level > 0: @@ -875,11 +1422,27 @@ def generate_topology(level: int, member: Dict[str, Any], yield member for member in members: - for member in generate_topology(level + 1, member, topology): - yield member + yield from generate_topology(level + 1, member, topology) def topology_sort(members: List[Dict[str, Any]]) -> Iterator[Dict[str, Any]]: + """Sort *members* according to their level in the replication topology tree. + + :param members: list of members in the cluster. Each item should countain at least these keys: + + * ``name``: name of the node, according to ``name`` configuration; + * ``role``: ``leader``, ``standby_leader`` or ``replica``. + + Cascading replicas are identified through ``tags`` -> ``replicatefrom`` value -- if that is set, and they are + in fact attached to another replica. + + Besides ``name``, ``role`` and ``tags`` keys, it may contain other keys, which although ignored by this + function, will be yielded as part of the resulting object. The value of key ``name`` is changed through + :func:`generate_topology`. + + :yields: *members* sorted by level in the topology, and with a new ``name`` value according to their level + in the topology. + """ topology: Dict[str, List[Dict[str, Any]]] = defaultdict(list) leader = next((m for m in members if m['role'].endswith('leader')), {'name': None}) replicas = set(member['name'] for member in members if not member['role'].endswith('leader')) @@ -893,6 +1456,15 @@ def topology_sort(members: List[Dict[str, Any]]) -> Iterator[Dict[str, Any]]: def get_cluster_service_info(cluster: Dict[str, Any]) -> List[str]: + """Get complementary information about the cluster. + + :param cluster: a Patroni cluster represented as an object created through :func:`~patroni.utils.cluster_as_json`. + + :returns: a list of 0 or more informational messages. They can be about: + + * Cluster in maintenance mode; + * Scheduled switchovers. + """ service_info: List[str] = [] if cluster.get('pause'): service_info.append('Maintenance mode: on') @@ -908,6 +1480,39 @@ def get_cluster_service_info(cluster: Dict[str, Any]) -> List[str]: def output_members(obj: Dict[str, Any], cluster: Cluster, name: str, extended: bool = False, fmt: str = 'pretty', group: Optional[int] = None) -> None: + """Print information about the Patroni cluster and its members. + + Information is printed to console through :func:`print_output`, and contains: + + * ``Cluster``: name of the Patroni cluster, as per ``scope`` configuration; + * ``Member``: name of the Patroni node, as per ``name`` configuration; + * ``Host``: hostname (or IP) and port, as per ``postgresql.listen`` configuration; + * ``Role``: ``Leader``, ``Standby Leader``, ``Sync Standby`` or ``Replica``; + * ``State``: ``stopping``, ``stopped``, ``stop failed``, ``crashed``, ``running``, ``starting``, + ``start failed``, ``restarting``, ``restart failed``, ``initializing new cluster``, ``initdb failed``, + ``running custom bootstrap script``, ``custom bootstrap failed``, or ``creating replica``, and so on; + * ``TL``: current timeline in Postgres; + ``Lag in MB``: replication lag. + + Besides that it may also have: + * ``Group``: Citus group ID -- showed only if Citus is enabled. + * ``Pending restart``: if the node is pending a restart -- showed only if *extended*; + * ``Scheduled restart``: timestamp for scheduled restart, if any -- showed only if *extended*; + * ``Tags``: node tags, if any -- showed only if *extended*. + + The 3 extended columns are always included if *extended*, even if the member has no value for a given column. + If not *extended*, these columns may still be shown if any of the members has any information for them. + + :param obj: Patroni configuration. + :param cluster: Patroni cluster. + :param name: name of the Patroni cluster. + :param extended: if extended information (pending restarts, scheduled restarts, node tags) should be printed, if + available. + :param fmt: the output table printing format. See :func:`print_output` for available options. If *fmt* is neither + ``topology`` nor ``pretty``, then complementary information gathered through :func:`get_cluster_service_info` is + not printed. + :param group: filter which Citus group we should get members from. If ``None`` get from all groups. + """ rows: List[List[Any]] = [] logging.debug(cluster) @@ -983,6 +1588,21 @@ def output_members(obj: Dict[str, Any], cluster: Cluster, name: str, @click.pass_obj def members(obj: Dict[str, Any], cluster_names: List[str], group: Optional[int], fmt: str, watch: Optional[int], w: bool, extended: bool, ts: bool) -> None: + """Process ``list`` command of ``patronictl`` utility. + + Print information about the Patroni cluster through :func:`output_members`. + + :param obj: Patroni configuration. + :param cluster_names: name of clusters that should be printed. If ``None`` consider only the cluster present in + ``scope`` key of *obj*. + :param group: filter which Citus group we should get members from. Refer to the module note for more details. + :param fmt: the output table printing format. See :func:`print_output` for available options. + :param watch: if given print output every *watch* seconds. + :param w: if ``True`` print output every 2 seconds. + :param extended: if extended information should be printed. See ``extended`` argument of :func:`output_members` for + more details. + :param ts: if timestamp should be included in the output. + """ if not cluster_names: if 'scope' in obj: cluster_names = [obj['scope']] @@ -1007,10 +1627,28 @@ def members(obj: Dict[str, Any], cluster_names: List[str], group: Optional[int], @option_watchrefresh @click.pass_context def topology(ctx: click.Context, cluster_names: List[str], group: Optional[int], watch: Optional[int], w: bool) -> None: + """Process ``topology`` command of ``patronictl`` utility. + + Print information about the cluster in ``topology`` format through :func:`members`. + + :param ctx: click context to be passed to :func:`members`. + :param cluster_names: name of clusters that should be printed. See ``cluster_names`` argument of + :func:`output_members` for more details. + :param group: filter which Citus group we should get members from. See ``group`` argument of :func:`output_members` + for more details. + :param watch: if given print output every *watch* seconds. + :param w: if ``True`` print output every 2 seconds. + """ ctx.forward(members, fmt='topology') def timestamp(precision: int = 6) -> str: + """Get current timestamp with given *precision* as a string. + + :param precision: Amount of digits to be present in the precision. + + :returns: the current timestamp with given *precision*. + """ return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:precision - 7] @@ -1024,6 +1662,18 @@ def timestamp(precision: int = 6) -> str: @click.pass_obj def flush(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member_names: List[str], force: bool, role: str, target: str) -> None: + """Process ``flush`` command of ``patronictl`` utility. + + Discard scheduled restart or switchover events. + + :param obj: Patroni configuration. + :param cluster_name: name of the Patroni cluster. + :param group: filter which Citus group we should flush an event. Refer to the module note for more details. + :param member_names: name of the members which events should be flushed. + :param force: perform the operation without asking for confirmations. + :param role: role to filter members. See :func:`get_all_members` for available options. + :param target: the event that should be flushed -- ``restart`` or ``switchover``. + """ dcs = get_dcs(obj, cluster_name, group) cluster = dcs.get_cluster() @@ -1057,6 +1707,13 @@ def flush(obj: Dict[str, Any], cluster_name: str, group: Optional[int], def wait_until_pause_is_applied(dcs: AbstractDCS, paused: bool, old_cluster: Cluster) -> None: + """Wait for all members in the cluster to have ``pause`` state set to *paused*. + + :param dcs: DCS object from where to get fresh cluster information. + :param paused: the desired state for ``pause`` in all nodes. + :param old_cluster: original cluster information before pause or unpause has been requested. Used to report which + nodes are still pending to have ``pause`` equal *paused* at a given point in time. + """ from patroni.config import get_global_config config = get_global_config(old_cluster) @@ -1081,6 +1738,20 @@ def wait_until_pause_is_applied(dcs: AbstractDCS, paused: bool, old_cluster: Clu def toggle_pause(config: Dict[str, Any], cluster_name: str, group: Optional[int], paused: bool, wait: bool) -> None: + """Toggle the ``pause`` state in the cluster members. + + :param config: Patroni configuration. + :param cluster_name: name of the Patroni cluster. + :param group: filter which Citus group we should toggle the pause state of. Refer to the module note for more + details. + :param paused: the desired state for ``pause`` in all nodes. + :param wait: ``True`` if it should block until the operation is finished or ``false`` for returning immediately. + + :raises: + PatroniCtlException: if + * ``pause`` state is already *paused*; or + * cluster contains no accessible members. + """ from patroni.config import get_global_config dcs = get_dcs(config, cluster_name, group) cluster = dcs.get_cluster() @@ -1114,6 +1785,15 @@ def toggle_pause(config: Dict[str, Any], cluster_name: str, group: Optional[int] @click.pass_obj @click.option('--wait', help='Wait until pause is applied on all nodes', is_flag=True) def pause(obj: Dict[str, Any], cluster_name: str, group: Optional[int], wait: bool) -> None: + """Process ``pause`` command of ``patronictl`` utility. + + Put the cluster in maintenance mode. + + :param obj: Patroni configuration. + :param cluster_name: name of the Patroni cluster. + :param group: filter which Citus group we should pause. Refer to the module note for more details. + :param wait: ``True`` if it should block until the operation is finished or ``false`` for returning immediately. + """ return toggle_pause(obj, cluster_name, group, True, wait) @@ -1123,17 +1803,27 @@ def pause(obj: Dict[str, Any], cluster_name: str, group: Optional[int], wait: bo @click.option('--wait', help='Wait until pause is cleared on all nodes', is_flag=True) @click.pass_obj def resume(obj: Dict[str, Any], cluster_name: str, group: Optional[int], wait: bool) -> None: + """Process ``unpause`` command of ``patronictl`` utility. + + Put the cluster out of maintenance mode. + + :param obj: Patroni configuration. + :param cluster_name: name of the Patroni cluster. + :param group: filter which Citus group we should unpause. Refer to the module note for more details. + :param wait: ``True`` if it should block until the operation is finished or ``false`` for returning immediately. + """ return toggle_pause(obj, cluster_name, group, False, wait) @contextmanager def temporary_file(contents: bytes, suffix: str = '', prefix: str = 'tmp') -> Generator[str, None, None]: - """Creates a temporary file with specified contents that persists for the context. + """Create a temporary file with specified contents that persists for the context. :param contents: binary string that will be written to the file. :param prefix: will be prefixed to the filename. :param suffix: will be appended to the filename. - :returns path of the created file. + + :yields: path of the created file. """ tmp = tempfile.NamedTemporaryFile(suffix=suffix, prefix=prefix, delete=False) with tmp: @@ -1146,9 +1836,23 @@ def temporary_file(contents: bytes, suffix: str = '', prefix: str = 'tmp') -> Ge def show_diff(before_editing: str, after_editing: str) -> None: - """Shows a diff between two strings. + """Show a diff between two strings. - If the output is to a tty the diff will be colored. Inputs are expected to be unicode strings. + Inputs are expected to be unicode strings. + + If the output is to a tty the diff will be colored. + + .. note:: + If tty it requires a pager program, and uses first found among: + * Program given by ``PAGER`` environment variable; or + * ``less``; or + * ``more``. + + :param before_editing: string to be compared with *after_editing*. + :param after_editing: string to be compared with *before_editing*. + + :raises: + :class:`PatroniCtlException`: if no suitable pager can be found when printing diff output to a tty. """ def listify(string: str) -> List[str]: return [line + '\n' for line in string.rstrip('\n').split('\n')] @@ -1195,38 +1899,59 @@ def show_diff(before_editing: str, after_editing: str) -> None: def format_config_for_editing(data: Any, default_flow_style: bool = False) -> str: - """Formats configuration as YAML for human consumption. + """Format configuration as YAML for human consumption. - :param data: configuration as nested dictionaries - :returns unicode YAML of the configuration""" + :param data: configuration as nested dictionaries. + :param default_flow_style: passed down as ``default_flow_style`` argument of :func:`yaml.safe_dump`. + + :returns: unicode YAML of the configuration. + """ return yaml.safe_dump(data, default_flow_style=default_flow_style, encoding=None, allow_unicode=True, width=200) def apply_config_changes(before_editing: str, data: Dict[str, Any], kvpairs: List[str]) -> Tuple[str, Dict[str, Any]]: - """Applies config changes specified as a list of key-value pairs. + """Apply config changes specified as a list of key-value pairs. Keys are interpreted as dotted paths into the configuration data structure. Except for paths beginning with - `postgresql.parameters` where rest of the path is used directly to allow for PostgreSQL GUCs containing dots. + ``postgresql.parameters`` where rest of the path is used directly to allow for PostgreSQL GUCs containing dots. Values are interpreted as YAML values. - :param before_editing: human representation before editing - :param data: configuration datastructure - :param kvpairs: list of strings containing key value pairs separated by = - :returns tuple of human readable and parsed datastructure after changes + :param before_editing: human representation before editing. + :param data: configuration data structure. + :param kvpairs: list of strings containing key value pairs separated by ``=``. + + :returns: tuple of human-readable, parsed data structure after changes. + + :raises: + :class:`PatroniCtlException`: if any entry in *kvpairs* is ``None`` or not in the expected format. """ changed_data = copy.deepcopy(data) - def set_path_value(config: Dict[str, Any], path: List[str], value: Any, prefix: Tuple[str, ...] = ()): + def set_path_value(config: Dict[str, Any], path: List[str], value: Any, prefix: Tuple[str, ...] = ()) -> None: + """Recursively walk through *config* and update setting specified by *path* with *value*. + + :param config: configuration data structure with all settings found under *prefix* path. + :param path: dotted path split by dot as delimiter into a list. Used to control the recursive calls and identify + when a leaf node is reached. + :param value: value for configuration described by *path*. If ``None`` the configuration key is removed from + *config*. + :param prefix: previous parts of *path* that have already been opened by parent recursive calls. Used to know + if we are changing a Postgres related setting or not. *prefix* plus *path* compose the original *path* given + on the root call. + """ # Postgresql GUCs can't be nested, but can contain dots so we re-flatten the structure for this case if prefix == ('postgresql', 'parameters'): path = ['.'.join(path)] key = path[0] + # When *path* contains a single item it means we reached a leaf node in the configuration, so we can remove or + # update the configuration based on what has been requested by the user. if len(path) == 1: if value is None: config.pop(key, None) else: config[key] = value + # Otherwise we need to keep navigating down in the configuration structure. else: if not isinstance(config.get(key), dict): config[key] = {} @@ -1244,11 +1969,12 @@ def apply_config_changes(before_editing: str, data: Dict[str, Any], kvpairs: Lis def apply_yaml_file(data: Dict[str, Any], filename: str) -> Tuple[str, Dict[str, Any]]: - """Applies changes from a YAML file to configuration + """Apply changes from a YAML file to configuration. - :param data: configuration datastructure - :param filename: name of the YAML file, - is taken to mean standard input - :returns tuple of human readable and parsed datastructure after changes + :param data: configuration data structure. + :param filename: name of the YAML file, ``-`` is taken to mean standard input. + + :returns: tuple of human-readable and parsed data structure after changes. """ changed_data = copy.deepcopy(data) @@ -1264,12 +1990,24 @@ def apply_yaml_file(data: Dict[str, Any], filename: str) -> Tuple[str, Dict[str, def invoke_editor(before_editing: str, cluster_name: str) -> Tuple[str, Dict[str, Any]]: - """Starts editor command to edit configuration in human readable format + """Start editor command to edit configuration in human readable format. - :param before_editing: human representation before editing - :returns tuple of human readable and parsed datastructure after changes + .. note:: + Requires an editor program, and uses first found among: + * Program given by ``EDITOR`` environemnt variable; or + * ``editor``; or + * ``vi``. + + :param before_editing: human representation before editing. + :param cluster_name: name of the Patroni cluster. + + :returns: tuple of human-readable, parsed data structure after changes. + + :raises: + :class:`PatroniCtlException`: if + * No suitable editor can be found; or + * Editor call exits with unexpected return code. """ - editor_cmd = os.environ.get('EDITOR') if not editor_cmd: for editor in ('editor', 'vi'): @@ -1310,6 +2048,27 @@ def invoke_editor(before_editing: str, cluster_name: str) -> Tuple[str, Dict[str def edit_config(obj: Dict[str, Any], cluster_name: str, group: Optional[int], force: bool, quiet: bool, kvpairs: List[str], pgkvpairs: List[str], apply_filename: Optional[str], replace_filename: Optional[str]) -> None: + """Process ``edit-config`` command of ``patronictl`` utility. + + Update or replace Patroni configuration in the DCS. + + :param obj: Patroni configuration. + :param cluster_name: name of the Patroni cluster. + :param group: filter which Citus group configuration we should edit. Refer to the module note for more details. + :param force: if ``True`` apply config changes without asking for confirmations. + :param quiet: if ``True`` skip showing config diff in the console. + :param kvpairs: list of key value general parameters to be changed. + :param pgkvpairs: list of key value Postgres parameters to be changed. + :param apply_filename: name of the file which contains with new configuration parameters to be applied. Pass ``-`` + for using stdin instead. + :param replace_filename: name of the file which contains the new configuration parameters to replace the existing + configuration. Pass ``-`` for using stdin instead. + + :raises: + :class:`PatroniCtlException`: if + * Configuration is absent from DCS; or + * Detected a concurrent modification of the configuration in the DCS. + """ dcs = get_dcs(obj, cluster_name, group) cluster = dcs.get_cluster() @@ -1358,6 +2117,14 @@ def edit_config(obj: Dict[str, Any], cluster_name: str, group: Optional[int], @option_default_citus_group @click.pass_obj def show_config(obj: Dict[str, Any], cluster_name: str, group: Optional[int]) -> None: + """Process ``show-config`` command of ``patronictl`` utility. + + Show Patroni configuration stored in the DCS. + + :param obj: Patroni configuration. + :param cluster_name: name of the Patroni cluster. + :param group: filter which Citus group configuration we should show. Refer to the module note for more details. + """ cluster = get_dcs(obj, cluster_name, group).get_cluster() if cluster.config: click.echo(format_config_for_editing(cluster.config.data)) @@ -1369,6 +2136,18 @@ def show_config(obj: Dict[str, Any], cluster_name: str, group: Optional[int]) -> @option_citus_group @click.pass_obj def version(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member_names: List[str]) -> None: + """Process ``version`` command of ``patronictl`` utility. + + Show version of: + * ``patronictl`` on invoker; + * ``patroni`` on all members of the cluster; + * ``PostgreSQL`` on all members of the cluster. + + :param obj: Patroni configuration. + :param cluster_name: name of the Patroni cluster. + :param group: filter which Citus group we should get members from. Refer to the module note for more details. + :param member_names: filter which members we should get version information from. + """ click.echo("patronictl version {0}".format(__version__)) if not cluster_name: @@ -1396,6 +2175,22 @@ def version(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member @option_format @click.pass_obj def history(obj: Dict[str, Any], cluster_name: str, group: Optional[int], fmt: str) -> None: + """Process ``history`` command of ``patronictl`` utility. + + Show the history of failover/switchover events in the cluster. + + Information is printed to console through :func:`print_output`, and contains: + * ``TL``: Postgres timeline when the event occurred; + * ``LSN``: Postgres LSN, in bytes, when the event occurred; + * ``Reason``: the reason that motivated the event, if any; + * ``Timestamp``: timestamp when the event occurred; + * ``New Leader``: the Postgres node that was promoted during the event. + + :param obj: Patroni configuration. + :param cluster_name: name of the Patroni cluster. + :param group: filter which Citus group we should get events from. Refer to the module note for more details. + :param fmt: the output table printing format. See :func:`print_output` for available options. + """ cluster = get_dcs(obj, cluster_name, group).get_cluster() cluster_history = cluster.history.lines if cluster.history else [] history: List[List[Any]] = list(map(list, cluster_history)) @@ -1409,6 +2204,23 @@ def history(obj: Dict[str, Any], cluster_name: str, group: Optional[int], fmt: s def format_pg_version(version: int) -> str: + """Format Postgres version for human consumption. + + :param version: Postgres version represented as an integer. + + :returns: Postgres version represented as a human-readable string. + + :Example: + + >>> format_pg_version(90624) + '9.6.24' + + >>> format_pg_version(100000) + '10.0' + + >>> format_pg_version(140008) + '14.8' + """ if version < 100000: return "{0}.{1}.{2}".format(version // 10000, version // 100 % 100, version % 100) else: From 0cf20831613785396034cf88b0be72e8f745179f Mon Sep 17 00:00:00 2001 From: Israel Date: Tue, 6 Jun 2023 03:36:29 -0300 Subject: [PATCH 06/22] Add docstrings to `patroni.__init__` (#2698) References: PAT-111 --- patroni/__init__.py | 54 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/patroni/__init__.py b/patroni/__init__.py index 8b91c26c..7f7035c2 100644 --- a/patroni/__init__.py +++ b/patroni/__init__.py @@ -1,3 +1,10 @@ +"""Define general variables and functions for :mod:`patroni`. + +:var PATRONI_ENV_PREFIX: prefix for Patroni related configuration environment variables. +:var KUBERNETES_ENV_PREFIX: prefix for Kubernetes related configuration environment variables. +:var MIN_PSYCOPG2: minimum version of :mod:`psycopg2` required by Patroni to work. +""" + import sys from typing import Any, Callable, Iterator, Tuple @@ -8,12 +15,40 @@ MIN_PSYCOPG2 = (2, 5, 4) def fatal(string: str, *args: Any) -> None: - sys.stderr.write('FATAL: ' + string.format(*args) + '\n') - sys.exit(1) + """Write a fatal message to stderr and exit with code ``1``. + + :param string: message to be written before exiting. + """ + sys.exit('FATAL: ' + string.format(*args)) def parse_version(version: str) -> Tuple[int, ...]: + """Convert *version* from human-readable format to tuple of integers. + + .. note:: + Designed for easy comparison of software versions in Python. + + :param version: human-readable software version, e.g. ``2.5.4``. + + :returns: tuple of *version* parts, each part as an integer. + + :Example: + + >>> parse_version('2.5.4') + (2, 5, 4) + """ def _parse_version(version: str) -> Iterator[int]: + """Yield each part of a human-readable version string as an integer. + + :param version: human-readable software version, e.g. ``2.5.4``. + + :yields: each part of *version* as an integer. + + :Example: + + >>> tuple(_parse_version('2.5.4')) + (2, 5, 4) + """ for e in version.split('.'): try: yield int(e) @@ -22,9 +57,22 @@ def parse_version(version: str) -> Tuple[int, ...]: return tuple(_parse_version(version.split(' ')[0])) -# We pass MIN_PSYCOPG2 and parse_version as arguments to simplify usage of check_psycopg from the setup.py def check_psycopg(_min_psycopg2: Tuple[int, ...] = MIN_PSYCOPG2, _parse_version: Callable[[str], Tuple[int, ...]] = parse_version) -> None: + """Ensure at least one among :mod:`psycopg2` or :mod:`psycopg` libraries are available in the environment. + + .. note:: + We pass ``MIN_PSYCOPG2`` and :func:`parse_version` as arguments to simplify usage of :func:`check_psycopg` from + the ``setup.py``. + + .. note:: + Patroni chooses :mod:`psycopg2` over :mod:`psycopg`, if possible. + + If nothing meeting the requirements is found, then exit with a fatal message. + + :param _min_psycopg2: minimum required version in case :mod:`psycopg2` is chosen. + :param _parse_version: function used to parse :mod:`psycopg2`/:mod:`psycopg` version into a comparable object. + """ min_psycopg2_str = '.'.join(map(str, _min_psycopg2)) # try psycopg2 From 4e52d4bb2ebc5ca4af4f17460e3d0f993814bfc7 Mon Sep 17 00:00:00 2001 From: Israel Date: Tue, 6 Jun 2023 05:01:32 -0300 Subject: [PATCH 07/22] Add docstrings to `patroni.collections` (#2702) --- patroni/collections.py | 147 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 138 insertions(+), 9 deletions(-) diff --git a/patroni/collections.py b/patroni/collections.py index 6560c35d..c6d8f5d6 100644 --- a/patroni/collections.py +++ b/patroni/collections.py @@ -1,73 +1,202 @@ +"""Patroni custom object types somewhat like :mod:`collections` module. + +Provides a case insensitive :class:`dict` and :class:`set` object types. +""" from collections import OrderedDict from typing import Any, Collection, Dict, Iterator, MutableMapping, MutableSet, Optional class CaseInsensitiveSet(MutableSet[str]): - """A case-insensitive ``set``-like object. + """A case-insensitive :class:`set`-like object. - Implements all methods and operations of :class:``MutableSet``. All values are expected to be strings. + Implements all methods and operations of :class:`~typing.MutableSet`. All values are expected to be strings. The structure remembers the case of the last value set, however, contains testing is case insensitive. """ + def __init__(self, values: Optional[Collection[str]] = None) -> None: + """Create a new instance of :class:`CaseInsensitiveSet` with the given *values*. + + :param values: values to be added to the set. + """ self._values: Dict[str, str] = {} for v in values or (): self.add(v) def __repr__(self) -> str: + """Get a string representation of the set. + + Provide a helpful way of recreating the set. + + :returns: representation of the set, showing its values. + + :Example: + + >>> repr(CaseInsensitiveSet(('1', 'test', 'Test', 'TESt', 'test2'))) # doctest: +ELLIPSIS + "'.format(type(self).__name__, tuple(self._values.values()), id(self)) def __str__(self) -> str: + """Get set values for printing. + + :returns: set of values in string format. + + :Example: + + >>> str(CaseInsensitiveSet(('1', 'test', 'Test', 'TESt', 'test2'))) # doctest: +SKIP + "{'TESt', 'test2', '1'}" + """ return str(set(self._values.values())) def __contains__(self, value: str) -> bool: + """Check if set contains *value*. + + The check is performed case-insensitively. + + :param value: value to be checked. + + :returns: ``True`` if *value* is already in the set, ``False`` otherwise. + """ return value.lower() in self._values def __iter__(self) -> Iterator[str]: + """Iterate over the values in this set. + + :yields: values from set. + """ return iter(self._values.values()) def __len__(self) -> int: + """Get the length of this set. + + :returns: number of values in the set. + + :Example: + + >>> len(CaseInsensitiveSet(('1', 'test', 'Test', 'TESt', 'test2'))) + 3 + """ return len(self._values) def add(self, value: str) -> None: + """Add *value* to this set. + + Search is performed case-insensitively. If *value* is already in the set, overwrite it with *value*, so we + "remember" the last case of *value*. + + :param value: value to be added to the set. + """ self._values[value.lower()] = value def discard(self, value: str) -> None: + """Remove *value* from this set. + + Search is performed case-insensitively. If *value* is not present in the set, no exception is raised. + + :param value: value to be removed from the set. + """ self._values.pop(value.lower(), None) def issubset(self, other: 'CaseInsensitiveSet') -> bool: + """Check if this set is a subset of *other*. + + :param other: another set to be compared with this set. + :returns: ``True`` if this set is a subset of *other*, else ``False``. + """ return self <= other class CaseInsensitiveDict(MutableMapping[str, Any]): - """A case-insensitive ``dict``-like object. + """A case-insensitive :class:`dict`-like object. - Implements all methods and operations of :class:``MutableMapping`` as well as dict's :func:``copy``. - All keys are expected to be strings. The structure remembers the case of the last key to be set, - and ``iter(instance)``, ``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()`` will contain - case-sensitive keys. However, querying and contains testing is case insensitive. + Implements all methods and operations of :class:`~typing.MutableMapping` as well as :class:`dict`'s + :func:`~dict.copy`. All keys are expected to be strings. The structure remembers the case of the last key to be set, + and :func:`iter`, :func:`dict.keys`, :func:`dict.items`, :func:`dict.iterkeys`, and :func:`dict.iteritems` will + contain case-sensitive keys. However, querying and contains testing is case insensitive. """ + def __init__(self, data: Optional[Dict[str, Any]] = None) -> None: + """Create a new instance of :class:`CaseInsensitiveDict` with the given *data*. + + :param data: initial dictionary to create a :class:`CaseInsensitiveDict` from. + """ self._values: OrderedDict[str, Any] = OrderedDict() self.update(data or {}) def __setitem__(self, key: str, value: Any) -> None: - # Use the lowercase key for lookups, but store the actual key alongside the value. + """Assign *value* to *key* in this dict. + + *key* is searched/stored case-insensitively in the dict. The corresponding value in the dict is a tuple of: + * original *key*; + * *value*. + + :param key: key to be created or updated in the dict. + :param value: value for *key*. + """ self._values[key.lower()] = (key, value) def __getitem__(self, key: str) -> Any: + """Get the value corresponding to *key*. + + *key* is searched case-insensitively in the dict. + + .. note: + If *key* is not present in the dict, :class:`KeyError` will be triggered. + + :param key: key to be searched in the dict. + + :returns: value corresponding to *key*. + """ return self._values[key.lower()][1] - def __delitem__(self, key: str) -> Any: + def __delitem__(self, key: str) -> None: + """Remove *key* from this dict. + + *key* is searched case-insensitively in the dict. + + .. note: + If *key* is not present in the dict, :class:`KeyError` will be triggered. + + :param key: key to be removed from the dict. + """ del self._values[key.lower()] def __iter__(self) -> Iterator[str]: + """Iterate over keys of this dict. + + :yields: each key present in the dict. Yields each key with its last case that has been stored. + """ return iter(key for key, _ in self._values.values()) def __len__(self) -> int: + """Get the length of this dict. + + :returns: number of keys in the dict. + + :Example: + + >>> len(CaseInsensitiveDict({'a': 'b', 'A': 'B', 'c': 'd'})) + 2 + """ return len(self._values) def copy(self) -> 'CaseInsensitiveDict': + """Create a copy of this dict. + + :return: a new dict object with the same keys and values of this dict. + """ return CaseInsensitiveDict({v[0]: v[1] for v in self._values.values()}) def __repr__(self) -> str: + """Get a string representation of the dict. + + Provide a helpful way of recreating the dict. + + :returns: representation of the dict, showing its keys and values. + + :Example: + + >>> repr(CaseInsensitiveDict({'a': 'b', 'A': 'B', 'c': 'd'})) # doctest: +ELLIPSIS + "'.format(type(self).__name__, dict(self.items()), id(self)) From e9f9e1cfad6d8de1dcf1a21d2d8f5e25d462f182 Mon Sep 17 00:00:00 2001 From: Israel Date: Tue, 6 Jun 2023 05:02:44 -0300 Subject: [PATCH 08/22] Add docstrings to `patroni.exceptions` (#2703) --- patroni/exceptions.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/patroni/exceptions.py b/patroni/exceptions.py index f8bf6df1..20c4797a 100644 --- a/patroni/exceptions.py +++ b/patroni/exceptions.py @@ -1,33 +1,55 @@ +"""Implement high-level Patroni exceptions. + +More specific exceptions can be found in other modules, as subclasses of any exception defined in this module. +""" from typing import Any class PatroniException(Exception): + """Parent class for all kind of Patroni exceptions. - """Parent class for all kind of exceptions related to selected distributed configuration store""" + :ivar value: description of the exception. + """ def __init__(self, value: Any) -> None: + """Create a new instance of :class:`PatroniException` with the given description. + + :param value: description of the exception. + """ self.value = value class PatroniFatalException(PatroniException): + """Catastrophic exception that prevents Patroni from performing its job.""" + pass class PostgresException(PatroniException): + """Any exception related with Postgres management.""" + pass class DCSError(PatroniException): + """Parent class for all kind of DCS related exceptions.""" + pass class PostgresConnectionException(PostgresException): + """Any problem faced while connecting to a Postgres instance.""" + pass class WatchdogError(PatroniException): + """Any problem faced while managing a watchdog device.""" + pass class ConfigParseError(PatroniException): + """Any issue identified while loading or validating the Patroni configuration.""" + pass From bd951ccdefe328e2412311010be1ec43a7932eaf Mon Sep 17 00:00:00 2001 From: Israel Date: Fri, 9 Jun 2023 09:07:06 -0300 Subject: [PATCH 09/22] Add docstrings to `patroni.async_executor` (#2704) References: PAT-120. --- patroni/async_executor.py | 110 ++++++++++++++++++++++++++++++++++---- 1 file changed, 100 insertions(+), 10 deletions(-) diff --git a/patroni/async_executor.py b/patroni/async_executor.py index 39f11013..f7fac02a 100644 --- a/patroni/async_executor.py +++ b/patroni/async_executor.py @@ -1,3 +1,4 @@ +"""Implement facilities for executing asynchronous tasks.""" import logging from threading import Event, Lock, RLock, Thread @@ -13,14 +14,23 @@ class CriticalTask(object): """Represents a critical task in a background process that we either need to cancel or get the result of. Fields of this object may be accessed only when holding a lock on it. To perform the critical task the background - thread must, while holding lock on this object, check `is_cancelled` flag, run the task and mark the task as - complete using `complete()`. + thread must, while holding lock on this object, check ``is_cancelled`` flag, run the task and mark the task as + complete using :func:`complete`. The main thread must hold async lock to prevent the task from completing, hold lock on critical task object, - call cancel. If the task has completed `cancel()` will return False and `result` field will contain the result of - the task. When cancel returns True it is guaranteed that the background task will notice the `is_cancelled` flag. + call :func:`cancel`. If the task has completed :func:`cancel` will return ``False`` and ``result`` field will + contain the result of the task. When :func:`cancel` returns ``True`` it is guaranteed that the background task will + notice the ``is_cancelled`` flag. + + :ivar is_cancelled: if the critical task has been cancelled. + :ivar result: contains the result of the task, if it has already been completed. """ + def __init__(self) -> None: + """Create a new instance of :class:`CriticalTask`. + + Instantiate the lock and the task control attributes. + """ self._lock = Lock() self.is_cancelled = False self.result = None @@ -28,37 +38,59 @@ class CriticalTask(object): def reset(self) -> None: """Must be called every time the background task is finished. - Must be called from async thread. Caller must hold lock on async executor when calling.""" + .. note:: + Must be called from async thread. Caller must hold lock on async executor when calling. + """ self.is_cancelled = False self.result = None def cancel(self) -> bool: - """Tries to cancel the task, returns True if the task has already run. + """Tries to cancel the task. - Caller must hold lock on async executor and the task when calling.""" + .. note:: + Caller must hold lock on async executor and the task when calling. + + :returns: ``False`` if the task has already run, or ``True`` it has been cancelled. + """ if self.result is not None: return False self.is_cancelled = True return True def complete(self, result: Any) -> None: - """Mark task as completed along with a result. + """Mark task as completed along with a *result*. - Must be called from async thread. Caller must hold lock on task when calling.""" + .. note:: + Must be called from async thread. Caller must hold lock on task when calling. + """ self.result = result def __enter__(self) -> 'CriticalTask': + """Acquire the object lock when entering the context manager.""" self._lock.acquire() return self def __exit__(self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]) -> None: + """Release the object lock when exiting the context manager.""" self._lock.release() class AsyncExecutor(object): + """Asynchronous executor of (long) tasks. + + :ivar critical_task: a :class:`CriticalTask` instance to handle execution of critical background tasks. + """ def __init__(self, cancellable: CancellableSubprocess, ha_wakeup: Callable[..., None]) -> None: + """Create a new instance of :class:`AsyncExecutor`. + + Configure the given *cancellable* and *ha_wakeup*, initializes the control attributes, and instantiate the lock + and event objects that are used to access attributes and manage communication between threads. + + :param cancellable: a subprocess that supports being cancelled. + :param ha_wakeup: function to wake up the HA loop. + """ self._cancellable = cancellable self._ha_wakeup = ha_wakeup self._thread_lock = RLock() @@ -70,9 +102,22 @@ class AsyncExecutor(object): @property def busy(self) -> bool: + """``True`` if there is an action scheduled to occur, else ``False``.""" return self.scheduled_action is not None def schedule(self, action: str) -> Optional[str]: + """Schedule *action* to be executed. + + .. note:: + Must be called before executing a task. + + .. note:: + *action* can only be scheduled if there is no other action currently scheduled. + + :param action: action to be executed. + + :returns: ``None`` if *action* has been successfully scheduled, or the previously scheduled action, if any. + """ with self._scheduled_action_lock: if self._scheduled_action is not None: return self._scheduled_action @@ -83,14 +128,32 @@ class AsyncExecutor(object): @property def scheduled_action(self) -> Optional[str]: + """The currently scheduled action, if any, else ``None``.""" with self._scheduled_action_lock: return self._scheduled_action def reset_scheduled_action(self) -> None: + """Unschedule a previously scheduled action, if any. + + .. note:: + Must be called once the scheduled task finishes or is cancelled. + """ with self._scheduled_action_lock: self._scheduled_action = None - def run(self, func: Callable[..., Any], args: Tuple[Any, ...] = ()) -> Optional[bool]: + def run(self, func: Callable[..., Any], args: Tuple[Any, ...] = ()) -> Optional[Any]: + """Run *func* with *args*. + + .. note:: + Expected to be executed through a thread. + + :param func: function to be run. If it returns anything other than ``None``, HA loop will be woken up at the end + of :func:`run` execution. + :param args: arguments to be passed to *func*. + + :returns: ``None`` if *func* execution has been cancelled or faced any exception, otherwise the result of + *func*. + """ wakeup = False try: with self: @@ -114,15 +177,36 @@ class AsyncExecutor(object): self._ha_wakeup() def run_async(self, func: Callable[..., Any], args: Tuple[Any, ...] = ()) -> None: + """Start an async thread that runs *func* with *args*. + + :param func: function to be run. Will be passed through args to :class:`~threading.Thread` with a target of + :func:`run`. + :param args: arguments to be passed along to :class:`~threading.Thread` with *func*. + + """ Thread(target=self.run, args=(func, args)).start() def try_run_async(self, action: str, func: Callable[..., Any], args: Tuple[Any, ...] = ()) -> Optional[str]: + """Try to run an async task, if none is currently being executed. + + :param action: name of the task to be executed. + :param func: actual function that performs the task *action*. + :param args: arguments to be passed to *func*. + + :returns: ``None`` if *func* was scheduled successfully, otherwise an error message informing of an already + ongoing task. + """ prev = self.schedule(action) if prev is None: return self.run_async(func, args) return 'Failed to run {0}, {1} is already in progress'.format(action, prev) def cancel(self) -> None: + """Request cancellation of a scheduled async task, if any. + + .. note:: + Wait until task is cancelled before returning control to caller. + """ with self: with self._scheduled_action_lock: if self._scheduled_action is None: @@ -137,9 +221,15 @@ class AsyncExecutor(object): self.reset_scheduled_action() def __enter__(self) -> 'AsyncExecutor': + """Acquire the thread lock when entering the context manager.""" self._thread_lock.acquire() return self def __exit__(self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]) -> None: + """Release the thread lock when exiting the context manager. + + .. note:: + The arguments are not used, but we need them to match the expected method signature. + """ self._thread_lock.release() From 2354f8f0043a28425814428ebe5c6e9ff9e11a6e Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 12 Jun 2023 01:52:46 -0400 Subject: [PATCH 10/22] Fix a few concurrency bugs in Citus support (#2710) - the `_in_fligh` attribute is accessed from multiple threads and must be protected with mutex when it is changed - allow adding tasks for `_in_fligh.group` from the `sync_pg_dist_node()` method when timeout is reached. Not doing so might result is indefinite transaction if REST API request from worker node failed. --- patroni/postgresql/citus.py | 56 ++++++++++++++++++++++++------------- tests/test_citus.py | 8 ++++-- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/patroni/postgresql/citus.py b/patroni/postgresql/citus.py index 740369ef..e11c206c 100644 --- a/patroni/postgresql/citus.py +++ b/patroni/postgresql/citus.py @@ -76,8 +76,9 @@ class CitusHandler(Thread): self._connection = Connection() self._pg_dist_node: Dict[int, PgDistNode] = {} # Cache of pg_dist_node: {groupid: PgDistNode()} self._tasks: List[PgDistNode] = [] # Requests to change pg_dist_node, every task is a `PgDistNode` - self._condition = Condition() # protects _pg_dist_node, _tasks, and _schedule_load_pg_dist_node self._in_flight: Optional[PgDistNode] = None # Reference to the `PgDistNode` being changed in a transaction + self._schedule_load_pg_dist_node = True # Flag that "pg_dist_node" should be queried from the database + self._condition = Condition() # protects _pg_dist_node, _tasks, _in_flight, and _schedule_load_pg_dist_node self.schedule_cache_rebuild() def is_enabled(self) -> bool: @@ -117,7 +118,8 @@ class CitusHandler(Thread): except Exception as e: logger.error('Exception when executing query "%s", (%s): %r', sql, params, e) self._connection.close() - self._in_flight = None + with self._condition: + self._in_flight = None self.schedule_cache_rebuild() raise e @@ -215,17 +217,20 @@ class CitusHandler(Thread): def process_task(self, task: PgDistNode) -> bool: """Updates a single row in `pg_dist_node` table, optionally in a transaction. - The transaction is started if we do a demote of the worker node - or before promoting the other worker if there is not transaction - in progress. And, the transaction it is committed when the - switchover/failover completed. + The transaction is started if we do a demote of the worker node or before promoting the other worker if + there is no transaction in progress. And, the transaction is committed when the switchover/failover completed. - This method returns `True` if node was updated (optionally, - transaction was committed) as an indicator that - the `self._pg_dist_node` cache should be updated. + .. note: + The maximum lifetime of the transaction in progress is controlled outside of this method. - The maximum lifetime of the transaction in progress - is controlled outside of this method.""" + .. note: + Read access to `self._in_flight` isn't protected because we know it can't be changed outside of our thread. + + :param task: reference to a :class:`PgDistNode` object that represents a row to be updated/created. + :returns: `True` if the row was succesfully created/updated or transaction in progress + was committed as an indicator that the `self._pg_dist_node` cache should be updated, + or, if the new transaction was opened, this method returns `False`. + """ if task.event == 'after_promote': # The after_promote may happen without previous before_demote and/or @@ -236,7 +241,6 @@ class CitusHandler(Thread): self.update_node(task) if self._in_flight: self.query('COMMIT') - self._in_flight = None return True else: # before_demote, before_promote if task.timeout: @@ -244,11 +248,11 @@ class CitusHandler(Thread): if not self._in_flight: self.query('BEGIN') self.update_node(task) - self._in_flight = task return False def process_tasks(self) -> None: while True: + # Read access to `_in_flight` isn't protected because we know it can't be changed outside of our thread. if not self._in_flight and not self.load_pg_dist_node(): break @@ -259,11 +263,17 @@ class CitusHandler(Thread): update_cache = self.process_task(task) except Exception as e: logger.error('Exception when working with pg_dist_node: %r', e) - update_cache = False + update_cache = None with self._condition: if self._tasks: if update_cache: self._pg_dist_node[task.group] = task + + if update_cache is False: # an indicator that process_tasks has started a transaction + self._in_flight = task + else: + self._in_flight = None + if id(self._tasks[i]) == id(task): self._tasks.pop(i) task.wakeup() @@ -293,11 +303,19 @@ class CitusHandler(Thread): with self._condition: i = self.find_task_by_group(task.group) - # task.timeout is None is an indicator that it was scheduled - # from the sync_pg_dist_node() and we don't want to override - # already existing task created from REST API. - if task.timeout is None and (i is not None or self._in_flight and self._in_flight.group == task.group): - return False + # The `PgDistNode.timeout` == None is an indicator that it was scheduled from the sync_pg_dist_node(). + if task.timeout is None: + # We don't want to override the already existing task created from REST API. + if i is not None and self._tasks[i].timeout is not None: + return False + + # There is a little race condition with tasks created from REST API - the call made "before" the member + # key is updated in DCS. Therefore it is possible that :func:`sync_pg_dist_node` will try to create a + # task based on the outdated values of "state"/"role". To solve it we introduce an artificial timeout. + # Only when the timeout is reached new tasks could be scheduled from sync_pg_dist_node() + if self._in_flight and self._in_flight.group == task.group and self._in_flight.timeout is not None\ + and self._in_flight.deadline > time.time(): + return False # Override already existing task for the same worker group if i is not None: diff --git a/tests/test_citus.py b/tests/test_citus.py index a2d096ba..40d4df8a 100644 --- a/tests/test_citus.py +++ b/tests/test_citus.py @@ -1,3 +1,4 @@ +import time from mock import Mock, patch from patroni.postgresql.citus import CitusHandler @@ -16,7 +17,7 @@ class TestCitus(BaseTestPostgresql): self.cluster = get_cluster_initialized_with_leader() self.cluster.workers[1] = self.cluster - @patch('time.time', Mock(side_effect=[100, 130, 160, 190, 220, 250, 280, 310])) + @patch('time.time', Mock(side_effect=[100, 130, 160, 190, 220, 250, 280, 310, 340, 370])) @patch('patroni.postgresql.citus.logger.exception', Mock(side_effect=SleepException)) @patch('patroni.postgresql.citus.logger.warning') @patch('patroni.postgresql.citus.PgDistNode.wait', Mock()) @@ -66,11 +67,14 @@ class TestCitus(BaseTestPostgresql): mock_logger.assert_called_once() self.assertTrue(mock_logger.call_args[0][0].startswith('Overriding existing task:')) - # add_task called from sync_pg_dist_node should not override already scheduled or in flight task + # add_task called from sync_pg_dist_node should not override already scheduled or in flight task until deadline self.assertIsNotNone(self.c.add_task('after_promote', 1, 'postgres://host:5432/postgres', 30)) self.assertIsNone(self.c.add_task('after_promote', 1, 'postgres://host:5432/postgres')) self.c._in_flight = self.c._tasks.pop() + self.c._in_flight.deadline = self.c._in_flight.timeout + time.time() self.assertIsNone(self.c.add_task('after_promote', 1, 'postgres://host:5432/postgres')) + self.c._in_flight.deadline = 0 + self.assertIsNotNone(self.c.add_task('after_promote', 1, 'postgres://host:5432/postgres')) # If there is no transaction in progress and cached pg_dist_node matching desired state task should not be added self.c._schedule_load_pg_dist_node = False From 9b01041175ad48956866006444918a70965b462b Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 21 Jun 2023 03:48:55 -0400 Subject: [PATCH 11/22] Compatibility with python 3.11 (#2718) now it checks that inside square brackets there is indeed IPv6. In addition to that fix a little issue in the function itself so it returns exactly the same result as psycopg2.extensions.parse_dsn(). Close https://github.com/zalando/patroni/pull/2714 --- patroni/postgresql/config.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/patroni/postgresql/config.py b/patroni/postgresql/config.py index 224b48ef..ef3f5c36 100644 --- a/patroni/postgresql/config.py +++ b/patroni/postgresql/config.py @@ -39,13 +39,14 @@ def conninfo_uri_parse(dsn: str) -> Dict[str, str]: for netloc in r.netloc.split('@')[-1].split(','): host = None if '[' in netloc and ']' in netloc: - host = netloc.split(']')[0][1:] - tmp = netloc.split(':', 1) + tmp = netloc.split(']') + [''] + host = tmp[0][1:] + netloc = ':'.join(tmp[:2]) + tmp = netloc.rsplit(':', 1) if host is None: host = tmp[0] hosts.append(host) - if len(tmp) == 2: - ports.append(tmp[1]) + ports.append(tmp[1] if len(tmp) == 2 else '') if hosts: ret['host'] = ','.join(hosts) if ports: @@ -113,9 +114,9 @@ def parse_dsn(value: str) -> Optional[Dict[str, str]]: and sets the `sslmode`, 'gssencmode', and `channel_binding` to `prefer` if it is not present in the connection string. This is necessary to simplify comparison of the old and the new values. - >>> r = parse_dsn('postgresql://u%2Fse:pass@:%2f123,[%2Fhost2]/db%2Fsdf?application_name=mya%2Fpp&ssl=true') - >>> r == {'application_name': 'mya/pp', 'host': ',/host2', 'sslmode': 'require',\ - 'password': 'pass', 'port': '/123', 'user': 'u/se', 'gssencmode': 'prefer', 'channel_binding': 'prefer'} + >>> r = parse_dsn('postgresql://u%2Fse:pass@:%2f123,[::1]/db%2Fsdf?application_name=mya%2Fpp&ssl=true') + >>> r == {'application_name': 'mya/pp', 'host': ',::1', 'sslmode': 'require',\ + 'password': 'pass', 'port': '/123,', 'user': 'u/se', 'gssencmode': 'prefer', 'channel_binding': 'prefer'} True >>> r = parse_dsn(" host = 'host' dbname = db\\\\ name requiressl=1 ") >>> r == {'host': 'host', 'sslmode': 'require', 'gssencmode': 'prefer', 'channel_binding': 'prefer'} From a00ffcb1a6520f714615f63d37110fe70095e270 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 21 Jun 2023 03:58:29 -0400 Subject: [PATCH 12/22] Update GUC's validator for PG16 beta1 (#2716) Plenty of GUC's were added, and some removed. `force_parallel_mode` was renamed to `debug_parallel_query`, but it is not worth special handling. --- .../available_parameters/0_postgres.yml | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/patroni/postgresql/available_parameters/0_postgres.yml b/patroni/postgresql/available_parameters/0_postgres.yml index 5afd2efb..056f7106 100644 --- a/patroni/postgresql/available_parameters/0_postgres.yml +++ b/patroni/postgresql/available_parameters/0_postgres.yml @@ -279,6 +279,9 @@ parameters: version_from: 90300 min_val: 0 max_val: 1.79769e+308 + createrole_self_grant: + - type: String + version_from: 160000 cursor_tuple_fraction: - type: Real version_from: 90300 @@ -307,6 +310,14 @@ parameters: version_from: 150000 min_val: 0 max_val: 0 + debug_io_direct: + - type: String + version_from: 160000 + debug_parallel_query: + - type: EnumBool + version_from: 160000 + possible_values: + - regress debug_pretty_print: - type: Bool version_from: 90300 @@ -437,6 +448,9 @@ parameters: enable_partitionwise_join: - type: Bool version_from: 110000 + enable_presorted_aggregate: + - type: Bool + version_from: 160000 enable_seqscan: - type: Bool version_from: 90300 @@ -469,6 +483,7 @@ parameters: force_parallel_mode: - type: EnumBool version_from: 90600 + version_till: 160000 possible_values: - regress from_collapse_limit: @@ -526,6 +541,9 @@ parameters: min_val: 64 max_val: 2147483647 unit: kB + gss_accept_delegation: + - type: Bool + version_from: 160000 hash_mem_multiplier: - type: Real version_from: 130000 @@ -551,6 +569,20 @@ parameters: min_val: 0 max_val: 2147483647 unit: kB + icu_validation_level: + - type: Enum + version_from: 160000 + possible_values: + - disabled + - debug5 + - debug4 + - debug3 + - debug2 + - debug1 + - log + - notice + - warning + - error ident_file: - type: String version_from: 90300 @@ -840,6 +872,12 @@ parameters: log_truncate_on_rotation: - type: Bool version_from: 90300 + logical_replication_mode: + - type: Enum + version_from: 160000 + possible_values: + - buffered + - immediate maintenance_io_concurrency: - type: Integer version_from: 130000 @@ -881,6 +919,11 @@ parameters: version_from: 100000 min_val: 0 max_val: 262143 + max_parallel_apply_workers_per_subscription: + - type: Integer + version_from: 160000 + min_val: 0 + max_val: 1024 max_parallel_maintenance_workers: - type: Integer version_from: 110000 @@ -1118,15 +1161,31 @@ parameters: version_till: 110000 min_val: 0 max_val: 2147483647 + reserved_connections: + - type: Integer + version_from: 160000 + min_val: 0 + max_val: 262143 restart_after_crash: - type: Bool version_from: 90300 row_security: - type: Bool version_from: 90500 + scram_iterations: + - type: Integer + version_from: 160000 + min_val: 1 + max_val: 2147483647 search_path: - type: String version_from: 90300 + send_abort_for_crash: + - type: Bool + version_from: 160000 + send_abort_for_kill: + - type: Bool + version_from: 160000 seq_page_cost: - type: Real version_from: 90300 @@ -1424,6 +1483,12 @@ parameters: update_process_title: - type: Bool version_from: 90300 + vacuum_buffer_usage_limit: + - type: Integer + version_from: 160000 + min_val: 0 + max_val: 16777216 + unit: kB vacuum_cleanup_index_scale_factor: - type: Real version_from: 110000 @@ -1465,6 +1530,7 @@ parameters: vacuum_defer_cleanup_age: - type: Integer version_from: 90300 + version_till: 160000 min_val: 0 max_val: 1000000 vacuum_failsafe_age: @@ -1656,6 +1722,7 @@ recovery_parameters: promote_trigger_file: - type: String version_from: 120000 + version_till: 160000 recovery_end_command: - type: String version_from: 90300 From 43e2290fdff7cdee598f2abf9f18e8801df94202 Mon Sep 17 00:00:00 2001 From: Mark Pekala Date: Wed, 21 Jun 2023 02:45:56 -0700 Subject: [PATCH 13/22] More beginner-friendly introduction (#2712) Attempts to make progress on #2250 by rephrasing existing introduction and making sentences slightly shorter. --- README.rst | 2 +- docs/README.rst | 2 +- docs/index.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index c26daf9a..18be2b49 100644 --- a/README.rst +++ b/README.rst @@ -8,7 +8,7 @@ You can find a version of this documentation that is searchable and also easier There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation `__. -Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper `__, `etcd `__, `Consul `__ or `Kubernetes `__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful. +Patroni is a template for high availability (HA) PostgreSQL solutions using Python. For maximum accessibility, Patroni supports a variety of distributed configuration stores like `ZooKeeper `__, `etcd `__, `Consul `__ or `Kubernetes `__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in datacenters — or anywhere else — will hopefully find it useful. We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely. diff --git a/docs/README.rst b/docs/README.rst index 8e38c908..4e3e7529 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -4,7 +4,7 @@ Introduction ============ -Patroni originated as a fork of `Governor `__, the project from Compose. It includes plenty of new features. +Patroni is a template for high availability (HA) PostgreSQL solutions using Python. Patroni originated as a fork of `Governor `__, the project from Compose. It includes plenty of new features. For an example of a Docker-based deployment with Patroni, see `Spilo `__, currently in use at Zalando. diff --git a/docs/index.rst b/docs/index.rst index 87bd4fdb..e76ec6d5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,7 +6,7 @@ Introduction ============ -Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper `__, `etcd `__, `Consul `__ or `Kubernetes `__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful. +Patroni is a template for high availability (HA) PostgreSQL solutions using Python. For maximum accessibility, Patroni supports a variety of distributed configuration stores like `ZooKeeper `__, `etcd `__, `Consul `__ or `Kubernetes `__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in datacenters — or anywhere else — will hopefully find it useful. We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely. There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation `__. From 6f91f4f4e2aa61296a26a82aaaaa685254f2d46c Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 22 Jun 2023 04:46:02 -0400 Subject: [PATCH 14/22] Release v3.0.3 (#2719) * Bump version * Bump pyright version and fix newly reported issues * Update release notes * Fix typos, extend release process desc * Add readthedocs configuration file v2 * Fix Dockerfile.citus files --- .github/workflows/tests.yaml | 2 +- .readthedocs.yaml | 21 ++++++++++++++ Dockerfile.citus | 2 +- docs/releases.rst | 54 ++++++++++++++++++++++++++++++++++++ kubernetes/Dockerfile.citus | 2 +- patroni/dcs/consul.py | 2 +- patroni/dcs/kubernetes.py | 7 +++-- patroni/version.py | 2 +- release.sh | 11 ++++---- 9 files changed, 90 insertions(+), 13 deletions(-) create mode 100644 .readthedocs.yaml diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index e35a2a81..78cce057 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -173,4 +173,4 @@ jobs: - uses: jakebailey/pyright-action@v1 with: - version: 1.1.309 + version: 1.1.315 diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..724e2418 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,21 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +formats: + - epub + - pdf + - htmlzip diff --git a/Dockerfile.citus b/Dockerfile.citus index 36dcbb51..8693b504 100644 --- a/Dockerfile.citus +++ b/Dockerfile.citus @@ -25,7 +25,7 @@ RUN set -ex \ | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \ | xargs apt-get install -y vim curl less jq locales haproxy sudo \ python3-etcd python3-kazoo python3-pip busybox \ - net-tools iputils-ping --fix-missing \ + net-tools iputils-ping lsb-release --fix-missing \ && if [ $(dpkg --print-architecture) = 'arm64' ]; then \ apt-get install -y postgresql-server-dev-$PG_MAJOR \ git gcc make autoconf \ diff --git a/docs/releases.rst b/docs/releases.rst index 753514be..b9b0dfa6 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -3,6 +3,60 @@ Release notes ============= +Version 3.0.3 +------------- + +**New features** + +- Compatibility with PostgreSQL 16 beta1 (Alexander Kukushkin) + + Extended GUC's validator rules. + +- Make PostgreSQL GUC's validator extensible (Israel Barth Rubio) + + Validator rules are loaded from YAML files located in ``patroni/postgresql/available_parameters/`` directory. Files are ordered in alphabetical order and applied one after another. It makes possible to have custom validators for non-standard Postgres distributions. + +- Added ``restapi.request_queue_size`` option (Andrey Zhidenkov) + + Sets request queue size for TCP socket used by Patroni REST API. Once the queue is full, further requests get a "Connection denied" error. The default value is 5. + +- Call ``initdb`` directly when initializing a new cluster (Matt Baker) + + Previously it was called via ``pg_ctl``, what required a special quoting of parameters passed to ``initdb``. + +- Added before stop hook (Le Duane) + + The hook could be configured via ``postgresql.before_stop`` and is executed right before ``pg_ctl stop``. The exit code doesn't impact shutdown process. + +- Added support for custom Postgres binary names (Israel Barth Rubio, Polina Bungina) + + When using a custom Postgres distribution it may be the case that the Postgres binaries are compiled with different names other than the ones used by the community Postgres distribution. Custom binary names could be configured using ``postgresql.bin_name.*`` and ``PATRONI_POSTGRESQL_BIN_*`` environment variables. + + +**Improvements** + +- Various improvements of ``patroni --validate-config`` (Polina Bungina) + + - Make ``bootstrap.initdb`` optional. It is only required for new clusters, but ``patroni --validate-config`` was complaining if it was missing in the config. + - Don't error out when ``postgresql.bin_dir`` is empty or not set. Try to first find Postgres binaries in the default PATH instead. + - Make ``postgresql.authentication.rewind`` section optional. If it is missing, Patroni is using the superuser. + +- Improved error reporting in ``patronictl`` (Israel Barth Rubio) + + The ``\n`` symbol was rendered as it is, instead of the actual newline symbol. + + +**Bugfixes** + +- Fixed issue in Citus support (Alexander Kukushkin) + + If the REST API call from the promoted worker to the coordinator failed during switchover it was leaving the given Citus group blocked during indefinite time. + +- Allow `etcd3` URL in `--dcs-url` option of `patronictl` (Israel Barth Rubio) + + If users attempted to pass a `etcd3` URL through `--dcs-url` option of `patronictl` they would face an exception. + + Version 3.0.2 ------------- diff --git a/kubernetes/Dockerfile.citus b/kubernetes/Dockerfile.citus index e61850c7..1ae242bf 100644 --- a/kubernetes/Dockerfile.citus +++ b/kubernetes/Dockerfile.citus @@ -7,7 +7,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \ && apt-get upgrade -y \ && apt-cache depends patroni | sed -n -e 's/.* Depends: \(python3-.\+\)$/\1/p' \ | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \ - | xargs apt-get install -y busybox vim-tiny curl jq less locales git python3-pip python3-wheel \ + | xargs apt-get install -y busybox vim-tiny curl jq less locales git python3-pip python3-wheel lsb-release \ ## Make sure we have a en_US.UTF-8 locale available && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ && echo "deb [signed-by=/etc/apt/trusted.gpg.d/citusdata_community.gpg] https://packagecloud.io/citusdata/community/debian/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/citusdata_community.list \ diff --git a/patroni/dcs/consul.py b/patroni/dcs/consul.py index 5327be0f..a62e7747 100644 --- a/patroni/dcs/consul.py +++ b/patroni/dcs/consul.py @@ -67,7 +67,7 @@ class HTTPClient(object): if ca_cert: kwargs['ca_certs'] = ca_cert kwargs['cert_reqs'] = ssl.CERT_REQUIRED if verify or ca_cert else ssl.CERT_NONE - self.http = urllib3.PoolManager(num_pools=10, maxsize=10, **kwargs) + self.http = urllib3.PoolManager(num_pools=10, maxsize=10, headers={}, **kwargs) self._ttl = 30 def set_read_timeout(self, timeout: float) -> None: diff --git a/patroni/dcs/kubernetes.py b/patroni/dcs/kubernetes.py index 0d457ba4..498cfd0f 100644 --- a/patroni/dcs/kubernetes.py +++ b/patroni/dcs/kubernetes.py @@ -75,7 +75,7 @@ class K8sConfig(object): pass def __init__(self) -> None: - self.pool_config: Dict[str, Union[str, int]] = {'maxsize': 10, 'num_pools': 10} # urllib3.PoolManager config + self.pool_config: Dict[str, Any] = {'maxsize': 10, 'num_pools': 10} # urllib3.PoolManager config self._token_expires_at = datetime.datetime.max self._headers: Dict[str, str] = {} self._make_headers() @@ -277,12 +277,13 @@ class K8sClient(object): def _get_api_servers(self, api_servers_cache: List[str]) -> List[str]: _, per_node_timeout, per_node_retries = self._calculate_timeouts(len(api_servers_cache)) - kwargs = {'headers': self._make_headers({}), 'preload_content': True, 'retries': per_node_retries, + headers = self._make_headers({}) + kwargs = {'preload_content': True, 'retries': per_node_retries, 'timeout': urllib3.Timeout(connect=max(1.0, per_node_timeout / 2.0), total=per_node_timeout)} path = self._API_URL_PREFIX + 'default/endpoints/kubernetes' for base_uri in api_servers_cache: try: - response = self.pool_manager.request('GET', base_uri + path, **kwargs) + response = self.pool_manager.request('GET', base_uri + path, headers=headers, **kwargs) endpoint = self._handle_server_response(response, True) if TYPE_CHECKING: # pragma: no cover assert isinstance(endpoint, K8sObject) diff --git a/patroni/version.py b/patroni/version.py index 4eaef889..96c68e77 100644 --- a/patroni/version.py +++ b/patroni/version.py @@ -2,4 +2,4 @@ :var __version__: the current Patroni version. """ -__version__ = '3.0.2' +__version__ = '3.0.3' diff --git a/release.sh b/release.sh index 95e6f572..6dd1f02c 100755 --- a/release.sh +++ b/release.sh @@ -1,11 +1,12 @@ #!/bin/bash # Release process: -# 1. Open a PR that updates release notes and Patroni version -# 2. Merge it -# 3. Run release.sh -# 4. After the new tag is pushed, the .github/workflows/release.yaml will run tests and upload the new package to test.pypi.org -# 5. Once the release is created, the .github/workflows/release.yaml will run tests and upload the new package to pypi.org +# 1. Open a PR that updates release notes, Patroni version and pyright version in the tests workflow. +# 2. Resolve possible typing issues. +# 3. Merge the PR. +# 4. Run release.sh +# 5. After the new tag is pushed, the .github/workflows/release.yaml will run tests and upload the new package to test.pypi.org +# 6. Once the release is created, the .github/workflows/release.yaml will run tests and upload the new package to pypi.org ## Bail out on any non-zero exitcode from the called processes set -xe From 74d78dbba258608cb9f8fa32e6329685ea85d047 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 26 Jun 2023 09:11:09 +0300 Subject: [PATCH 15/22] Update request_queue_size feature authors (#2723) Add Aleksei Sukhov do the authors --- docs/releases.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/releases.rst b/docs/releases.rst index b9b0dfa6..6adda2e1 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -16,7 +16,7 @@ Version 3.0.3 Validator rules are loaded from YAML files located in ``patroni/postgresql/available_parameters/`` directory. Files are ordered in alphabetical order and applied one after another. It makes possible to have custom validators for non-standard Postgres distributions. -- Added ``restapi.request_queue_size`` option (Andrey Zhidenkov) +- Added ``restapi.request_queue_size`` option (Andrey Zhidenkov, Aleksei Sukhov) Sets request queue size for TCP socket used by Patroni REST API. Once the queue is full, further requests get a "Connection denied" error. The default value is 5. From ed02826103a04effa098df6a9cfb647c6d974d45 Mon Sep 17 00:00:00 2001 From: Israel Date: Tue, 4 Jul 2023 12:53:24 -0300 Subject: [PATCH 16/22] REST API would not reload SSL certificate upon receiving an SIGHUP (#2722) Revert to using `ssl._ssl._test_decode_cert` A change has been included as part of Patroni 3.0.3 release: use public functions instead of `ssl._ssl._test_decode_cert` to get serial number of certificates. There was a slight bug in that implementation: it was only loading the certificates through `load_verify_locations`, but was missing to get the certificates through `get_ca_certs`. As a consequence Patroni was not able anymore to reload REST API cert on SIGHUP. An attempt to fix that issue was made through commit `20f578f09f3aa604e5288710d4fd4e611152ed5f`. However, even with the correct call of `get_ca_certs`, it was detected a corner case where `load_verify_locations` would skip loading a certificate: if it was issued with `CA:FALSE`. That essentially means the implementation is still buggy in that situation. See [CPython](https://github.com/python/cpython/blob/c283a0cff5603540f06d9017e484b3602cc62e7c/Modules/_ssl.c#L4618C14-L4619) for the underlying problem. In order to get back a functional implementation again we are reverting the code to use the private function `ssl._ssl._test_decode_cert`. We can later study a possible more elegant alternative for solving this, if any. --------- Signed-off-by: Israel Barth Rubio --- patroni/api.py | 10 +++++----- tests/test_api.py | 5 +---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/patroni/api.py b/patroni/api.py index f461811b..7be2a049 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -1541,11 +1541,11 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread): if self.__ssl_options.get('certfile'): import ssl try: - ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - crts = ctx.load_verify_locations(self.__ssl_options['certfile']) - if crts: - return crts[0].get('serialNumber') - except Exception as e: + crt: Dict[str, Any] = ssl._ssl._test_decode_cert(self.__ssl_options['certfile']) # pyright: ignore + if TYPE_CHECKING: # pragma: no cover + assert isinstance(crt, dict) + return crt.get('serialNumber') + except ssl.SSLError as e: logger.error('Failed to get serial number from certificate %s: %r', self.__ssl_options['certfile'], e) def reload_local_certificate(self) -> Optional[bool]: diff --git a/tests/test_api.py b/tests/test_api.py index 166d3eb1..25342dca 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -180,7 +180,6 @@ class MockRestApiServer(RestApiServer): @patch('ssl.SSLContext.load_cert_chain', Mock()) @patch('ssl.SSLContext.wrap_socket', Mock(return_value=0)) -@patch('ssl.SSLContext.load_verify_locations', Mock(return_value=[Mock()])) @patch.object(HTTPServer, '__init__', Mock()) class TestRestApiHandler(unittest.TestCase): @@ -589,7 +588,6 @@ class TestRestApiServer(unittest.TestCase): @patch('ssl.SSLContext.load_cert_chain', Mock()) @patch('ssl.SSLContext.set_ciphers', Mock()) @patch('ssl.SSLContext.wrap_socket', Mock(return_value=0)) - @patch('ssl.SSLContext.load_verify_locations', Mock(return_value=[Mock()])) @patch.object(HTTPServer, '__init__', Mock()) def setUp(self): self.srv = MockRestApiServer(Mock(), '', {'listen': '*:8008', 'certfile': 'a', 'verify_client': 'required', @@ -652,10 +650,9 @@ class TestRestApiServer(unittest.TestCase): mock_get_request.return_value = (self.__create_socket(), ('127.0.0.1', 55555)) self.srv._handle_request_noblock() - @patch('ssl.SSLContext.load_verify_locations', Mock(return_value=[Mock()])) + @patch('ssl._ssl._test_decode_cert', Mock()) def test_reload_local_certificate(self): self.assertTrue(self.srv.reload_local_certificate()) - @patch('ssl.SSLContext.load_verify_locations', Mock(side_effect=Exception)) def test_get_certificate_serial_number(self): self.assertIsNone(self.srv.get_certificate_serial_number()) From e72d3ba79e92708c378f48eec84979349dd44197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Marqu=C3=A9s?= Date: Tue, 4 Jul 2023 12:53:53 -0300 Subject: [PATCH 17/22] Use full names for contributors in the release notes (#2725) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Until the last release, contributors' names were fully written on the first occurence during that release. This meant that if Alexander had four contributions in the release, we would use Alexander Kukushkin on the first item in the release, and on all the others just Alexander. This could, in some cases, create some confusion. For example, if there are more than one contributor with the same first name that has more than one contribution each. For this reason, in release 3.0.3, we used the full names of contributors on all the items from the release. This patch is to amend the old release notes and have each entry with the full name of the contributor. Also fix typo with 2 spaces between first name and last name in one bug fix Signed-off-by: Martín Marqués --- docs/releases.rst | 596 +++++++++++++++++++++++----------------------- 1 file changed, 298 insertions(+), 298 deletions(-) diff --git a/docs/releases.rst b/docs/releases.rst index 6adda2e1..b9a3eae7 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -74,18 +74,18 @@ Version 3.0.2 It makes pager configurable via ``PAGER`` environment variable, which overrides default ``less`` and ``more``. -- Make K8s retriable HTTP status code configurable (Alexander) +- Make K8s retriable HTTP status code configurable (Alexander Kukushkin) On some managed platforms it is possible to get status code ``401 Unauthorized``, which sometimes gets resolved after a few retries. **Improvements** -- Set ``hot_standby`` to ``off`` during custom bootstrap only if ``recovery_target_action`` is set to ``promote`` (Alexander) +- Set ``hot_standby`` to ``off`` during custom bootstrap only if ``recovery_target_action`` is set to ``promote`` (Alexander Kukushkin) It was necessary to make ``recovery_target_action=pause`` work correctly. -- Don't allow ``on_reload`` callback to kill other callbacks (Alexander) +- Don't allow ``on_reload`` callback to kill other callbacks (Alexander Kukushkin) ``on_start``/``on_stop``/``on_role_change`` are usually used to add/remove Virtual IP and ``on_reload`` should not interfere with them. @@ -100,15 +100,15 @@ Version 3.0.2 It didn't work for namespaces different from ``default``. -- Don't write to ``PGDATA`` if major version is not known (Alexander) +- Don't write to ``PGDATA`` if major version is not known (Alexander Kukushkin) If right after the start ``PGDATA`` was empty (maybe wasn't yet mounted), Patroni was making a false assumption about PostgreSQL version and falsely creating ``recovery.conf`` file even if the actual major version is v10+. -- Fixed bug with Citus metadata after coordinator failover (Alexander) +- Fixed bug with Citus metadata after coordinator failover (Alexander Kukushkin) The ``citus_set_coordinator_host()`` call doesn't cause metadata sync and the change was invisible on worker nodes. The issue is solved by switching to ``citus_update_node()``. -- Use etcd hosts listed in the config file as a fallback when all etcd nodes "failed" (Alexander) +- Use etcd hosts listed in the config file as a fallback when all etcd nodes "failed" (Alexander Kukushkin) The etcd cluster may change topology over time and Patroni tries to follow it. If at some point all nodes became unreachable Patroni will use a combination of nodes from the config plus the last known topology when trying to reconnect. @@ -142,7 +142,7 @@ This version adds integration with `Citus `__ and mak If the feature is enabled it will allow Patroni cluster to survive temporary DCS outages. You can find more details in the :ref:`documentation `. -- Citus support (Alexander, Polina, Jelte Fennema) +- Citus support (Alexander Kukushkin, Polina Bungina, Jelte Fennema) Patroni enables easy deployment and management of `Citus `__ clusters with HA. Please check :ref:`here ` page for more information. @@ -153,7 +153,7 @@ This version adds integration with `Citus `__ and mak Patroni will still write these logs, but only in DEBUG. -- Run only one monitoring query per HA loop (Alexander) +- Run only one monitoring query per HA loop (Alexander Kukushkin) It wasn't the case if synchronous replication is enabled. @@ -161,14 +161,14 @@ This version adds integration with `Citus `__ and mak If bootstrap failed Patroni used to rename $PGDATA folder with timestamp suffix. From now on the suffix will be ``.failed`` and if such folder exists it is removed before renaming. -- Improved check of synchronous replication connections (Alexander) +- Improved check of synchronous replication connections (Alexander Kukushkin) When the new host is added to the ``synchronous_standby_names`` it will be set as synchronous in DCS only when it managed to catch up with the primary in addition to ``pg_stat_replication.sync_state = 'sync'``. **Removed functionality** -- Remove ``patronictl scaffold`` (Alexander) +- Remove ``patronictl scaffold`` (Alexander Kukushkin) The only reason for having it was a hacky way of running standby clusters. @@ -199,7 +199,7 @@ Version 2.1.6 **Security improvements** -- Enforce ``search_path=pg_catalog`` for non-replication connections (Alexander) +- Enforce ``search_path=pg_catalog`` for non-replication connections (Alexander Kukushkin) Since Patroni is heavily relying on superuser connections, we want to protect it from the possible attacks carried out using user-defined functions and/or operators in ``public`` schema with the same name and signature as the corresponding objects in ``pg_catalog``. For that, ``search_path=pg_catalog`` is enforced for all connections created by Patroni (except replication connections). @@ -214,7 +214,7 @@ Version 2.1.6 As it is effectively a non-required option. -- Improve behaviour of the insecure option (Alexander) +- Improve behaviour of the insecure option (Alexander Kukushkin) Ctl's ``insecure`` option didn't work properly when client certificates were used for REST API requests. @@ -226,14 +226,14 @@ Version 2.1.6 Only add ``.exe`` to a file name if it has no extension yet. -- Fix Consul TTL setup (Alexander) +- Fix Consul TTL setup (Alexander Kukushkin) We used ``ttl/2.0`` when setting the value on the HTTPClient, but forgot to multiply the current value by 2 in the class' property. It was resulting in Consul TTL off by twice. **Removed functionality** -- Remove ``patronictl configure`` (Polina) +- Remove ``patronictl configure`` (Polina Bungina) There is no more need for a separate ``patronictl`` config creation. @@ -257,18 +257,18 @@ This version enhances compatibility with PostgreSQL 15 and declares Etcd v3 supp If used instead of ``GET`` Patroni will return only the HTTP Status Code. -- Support behave tests on Windows (Alexander) +- Support behave tests on Windows (Alexander Kukushkin) Emulate graceful Patroni shutdown (``SIGTERM``) on Windows by introduce the new REST API endpoint ``POST /sigterm``. -- Introduce ``postgresql.proxy_address`` (Alexander) +- Introduce ``postgresql.proxy_address`` (Alexander Kukushkin) It will be written to the member key in DCS as the ``proxy_url`` and could be used/useful for service discovery. **Stability improvements** -- Call ``pg_replication_slot_advance()`` from a thread (Alexander) +- Call ``pg_replication_slot_advance()`` from a thread (Alexander Kukushkin) On busy clusters with many logical replication slots the ``pg_replication_slot_advance()`` call was affecting the main HA loop and could result in the member key expiration. @@ -276,15 +276,15 @@ This version enhances compatibility with PostgreSQL 15 and declares Etcd v3 supp If the primary crashed and was down during considerable time, some WAL files could be missing from archive and from the new primary. There is a chance that ``pg_rewind`` could remove these WAL files from the old primary making it impossible to start it as a standby. By archiving ``ready`` WAL files we not only mitigate this problem but in general improving continues archiving experience. -- Ignore ``403`` errors when trying to create Kubernetes Service (Nick Hudson, Polina) +- Ignore ``403`` errors when trying to create Kubernetes Service (Nick Hudson, Polina Bungina) Patroni was spamming logs by unsuccessful attempts to create the service, which in fact could already exist. -- Improve liveness probe (Alexander) +- Improve liveness probe (Alexander Kukushkin) The liveness problem will start failing if the heartbeat loop is running longer than `ttl` on the primary or `2*ttl` on the replica. That will allow us to use it as an alternative for :ref:`watchdog ` on Kubernetes. -- Make sure only sync node tries to grab the lock when switchover (Alexander, Polina) +- Make sure only sync node tries to grab the lock when switchover (Alexander Kukushkin, Polina Bungina) Previously there was a slim chance that up-to-date async member could become the leader if the manual switchover was performed without specifying the target. @@ -292,15 +292,15 @@ This version enhances compatibility with PostgreSQL 15 and declares Etcd v3 supp Do not allow a create replica method that does not require a leader to be triggered while the cluster bootstrap is running. -- Compatibility with kazoo-2.9.0 (Alexander) +- Compatibility with kazoo-2.9.0 (Alexander Kukushkin) Depending on python version the ``SequentialThreadingHandler.select()`` method may raise ``TypeError`` and ``IOError`` exceptions if ``select()`` is called on the closed socket. -- Explicitly shut down SSL connection before socket shutdown (Alexander) +- Explicitly shut down SSL connection before socket shutdown (Alexander Kukushkin) Not doing it resulted in ``unexpected eof while reading`` errors with OpenSSL 3.0. -- Compatibility with `prettytable>=2.2.0` (Alexander) +- Compatibility with `prettytable>=2.2.0` (Alexander Kukushkin) Due to the internal API changes the cluster name header was shown on the incorrect line. @@ -311,23 +311,23 @@ This version enhances compatibility with PostgreSQL 15 and declares Etcd v3 supp In case of error get the new token and retry request. -- Fix bug in the ``GET /read-only-sync`` endpoint (Alexander) +- Fix bug in the ``GET /read-only-sync`` endpoint (Alexander Kukushkin) It was introduced in previous release and effectively never worked. -- Handle the case when data dir storage disappeared (Alexander) +- Handle the case when data dir storage disappeared (Alexander Kukushkin) Patroni is periodically checking that the PGDATA is there and not empty, but in case of issues with storage the ``os.listdir()`` is raising the ``OSError`` exception, breaking the heart-beat loop. -- Apply ``master_stop_timeout`` when waiting for user backends to close (Alexander) +- Apply ``master_stop_timeout`` when waiting for user backends to close (Alexander Kukushkin) Something that looks like user backend could be in fact a background worker (e.g., Citus Maintenance Daemon) that is failing to stop. -- Accept ``*:`` for ``postgresql.listen`` (Denis) +- Accept ``*:`` for ``postgresql.listen`` (Denis Laxalde) The ``patroni --validate-config`` was complaining about it being invalid. -- Timeouts fixes in Raft (Alexander) +- Timeouts fixes in Raft (Alexander Kukushkin) When Patroni or patronictl are starting they try to get Raft cluster topology from known members. These calls were made without proper timeouts. @@ -368,19 +368,19 @@ Version 2.1.4 **Stability improvements** -- Don't copy the logical replication slot to a replica if there is a configuration mismatch in the logical decoding setup with the primary (Alexander) +- Don't copy the logical replication slot to a replica if there is a configuration mismatch in the logical decoding setup with the primary (Alexander Kukushkin) A replica won't copy a logical replication slot from the primary anymore if the slot doesn't match the ``plugin`` or ``database`` configuration options. Previously, the check for whether the slot matches those configuration options was not performed until after the replica copied the slot and started with it, resulting in unnecessary and repeated restarts. -- Special handling of recovery configuration parameters for PostgreSQL v12+ (Alexander) +- Special handling of recovery configuration parameters for PostgreSQL v12+ (Alexander Kukushkin) While starting as replica Patroni should be able to update ``postgresql.conf`` and restart/reload if the leader address has changed by caching current parameters values instead of querying them from ``pg_settings``. -- Better handling of IPv6 addresses in the ``postgresql.listen`` parameters (Alexander) +- Better handling of IPv6 addresses in the ``postgresql.listen`` parameters (Alexander Kukushkin) Since the ``listen`` parameter has a port, people try to put IPv6 addresses into square brackets, which were not correctly stripped when there is more than one IP in the list. -- Use ``replication`` credentials when performing divergence check only on PostgreSQL v10 and older (Alexander) +- Use ``replication`` credentials when performing divergence check only on PostgreSQL v10 and older (Alexander Kukushkin) If ``rewind`` is enabled, Patroni will again use either ``superuser`` or ``rewind`` credentials on newer Postgres versions. @@ -395,15 +395,15 @@ Version 2.1.4 In certain cases Patroni was trying to pass it as numeric. -- Better handling of failed ``pg_rewind`` attempt (Alexander) +- Better handling of failed ``pg_rewind`` attempt (Alexander Kukushkin) If the primary becomes unavailable during ``pg_rewind``, ``$PGDATA`` will be left in a broken state. Following that, Patroni will remove the data directory even if this is not allowed by the configuration. -- Don't remove ``slots`` annotations from the leader ``ConfigMap``/``Endpoint`` when PostgreSQL isn't ready (Alexander) +- Don't remove ``slots`` annotations from the leader ``ConfigMap``/``Endpoint`` when PostgreSQL isn't ready (Alexander Kukushkin) If ``slots`` value isn't passed the annotation will keep the current value. -- Handle concurrency problem with K8s API watchers (Alexander) +- Handle concurrency problem with K8s API watchers (Alexander Kukushkin) Under certain (unknown) conditions watchers might become stale; as a result, ``attempt_to_acquire_leader()`` method could fail due to the HTTP status code 409. In that case we reset watchers connections and restart from scratch. @@ -427,33 +427,33 @@ Version 2.1.3 **Stability improvements** -- Compatibility with legacy ``psycopg2`` (Alexander) +- Compatibility with legacy ``psycopg2`` (Alexander Kukushkin) For example, the ``psycopg2`` installed from Ubuntu 18.04 packages doesn't have the ``UndefinedFile`` exception yet. -- Restart ``etcd3`` watcher if all Etcd nodes don't respond (Alexander) +- Restart ``etcd3`` watcher if all Etcd nodes don't respond (Alexander Kukushkin) If the watcher is alive the ``get_cluster()`` method continues returning stale information even if all Etcd nodes are failing. -- Don't remove the leader lock in the standby cluster while paused (Alexander) +- Don't remove the leader lock in the standby cluster while paused (Alexander Kukushkin) Previously the lock was maintained only by the node that was running as a primary and not a standby leader. **Bugfixes** -- Fixed bug in the standby-leader bootstrap (Alexander) +- Fixed bug in the standby-leader bootstrap (Alexander Kukushkin) Patroni was considering bootstrap as failed if Postgres didn't start accepting connections after 60 seconds. The bug was introduced in the 2.1.2 release. -- Fixed bug with failover to a cascading standby (Alexander) +- Fixed bug with failover to a cascading standby (Alexander Kukushkin) When figuring out which slots should be created on cascading standby we forgot to take into account that the leader might be absent. -- Fixed small issues in Postgres config validator (Alexander) +- Fixed small issues in Postgres config validator (Alexander Kukushkin) Integer parameters introduced in PostgreSQL v14 were failing to validate because min and max values were quoted in the validator.py -- Use replication credentials when checking leader status (Alexander) +- Use replication credentials when checking leader status (Alexander Kukushkin) It could be that the ``remove_data_directory_on_diverged_timelines`` is set, but there is no ``rewind_credentials`` defined and superuser access between nodes is not allowed. @@ -469,7 +469,7 @@ Version 2.1.3 It could happen if the synchronous mode is enabled and the DCS content was wiped out. -- Fix bug in divergence timeline check (Alexander) +- Fix bug in divergence timeline check (Alexander Kukushkin) Patroni was falsely assuming that timelines have diverged. For pg_rewind it didn't create any problem, but if pg_rewind is not allowed and the ``remove_data_directory_on_diverged_timelines`` is set, it resulted in reinitializing the former leader. @@ -487,7 +487,7 @@ Version 2.1.2 This field notes the last time (as unix epoch) a cluster member has successfully communicated with the DCS. This is useful to identify and/or analyze network partitions. -- Release the leader lock when ``pg_controldata`` reports "shut down" (Alexander) +- Release the leader lock when ``pg_controldata`` reports "shut down" (Alexander Kukushkin) To solve the problem of slow switchover/shutdown in case ``archive_command`` is slow/failing, Patroni will remove the leader key immediately after ``pg_controldata`` started reporting PGDATA as ``shut down`` cleanly and it verified that there is at least one replica that received all changes. If there are no replicas that fulfill this condition the leader key is not removed and the old behavior is retained, i.e. Patroni will keep updating the lock. @@ -502,15 +502,15 @@ Version 2.1.2 **Stability improvements** -- Delay the next attempt of recovery till next HA loop (Alexander) +- Delay the next attempt of recovery till next HA loop (Alexander Kukushkin) If Postgres crashed due to out of disk space (for example) and fails to start because of that Patroni is too eagerly trying to recover it flooding logs. -- Add log before demoting, which can take some time (Michael) +- Add log before demoting, which can take some time (Michael Banck) It can take some time for the demote to finish and it might not be obvious from looking at the logs what exactly is going on. -- Improve "I am" status messages (Michael) +- Improve "I am" status messages (Michael Banck) ``no action. I am a secondary ({0})`` vs ``no action. I am ({0}), a secondary`` @@ -518,62 +518,62 @@ Version 2.1.2 It is possible to specify ``wal_keep_segments`` as a string in the global :ref:`dynamic configuration ` and due to Python being a dynamically typed language the string was simply multiplied. Example: ``wal_keep_segments: "100"`` was converted to ``100100100100100100100100100100100100100100100100MB``. -- Allow switchover only to sync nodes when synchronous replication is enabled (Alexander) +- Allow switchover only to sync nodes when synchronous replication is enabled (Alexander Kukushkin) In addition to that do the leader race only against known synchronous nodes. -- Use cached role as a fallback when Postgres is slow (Alexander) +- Use cached role as a fallback when Postgres is slow (Alexander Kukushkin) In some extreme cases Postgres could be so slow that the normal monitoring query does not finish in a few seconds. The ``statement_timeout`` exception not being properly handled could lead to the situation where Postgres was not demoted on time when the leader key expired or the update failed. In case of such exception Patroni will use the cached ``role`` to determine whether Postgres is running as a primary. -- Avoid unnecessary updates of the member ZNode (Alexander) +- Avoid unnecessary updates of the member ZNode (Alexander Kukushkin) If no values have changed in the members data, the update should not happen. -- Optimize checkpoint after promote (Alexander) +- Optimize checkpoint after promote (Alexander Kukushkin) Avoid doing ``CHECKPOINT`` if the latest timeline is already stored in ``pg_control``. It helps to avoid unnecessary ``CHECKPOINT`` right after initializing the new cluster with ``initdb``. -- Prefer members without ``nofailover`` when picking sync nodes (Alexander) +- Prefer members without ``nofailover`` when picking sync nodes (Alexander Kukushkin) Previously sync nodes were selected only based on the replication lag, hence the node with ``nofailover`` tag had the same chances to become synchronous as any other node. That behavior was confusing and dangerous at the same time because in case of a failed primary the failover could not happen automatically. -- Remove duplicate hosts from the etcd machine cache (Michael) +- Remove duplicate hosts from the etcd machine cache (Michael Banck) Advertised client URLs in the etcd cluster could be misconfigured. Removing duplicates in Patroni in this case is a low-hanging fruit. **Bugfixes** -- Skip temporary replication slots while doing slot management (Alexander) +- Skip temporary replication slots while doing slot management (Alexander Kukushkin) Starting from v10 ``pg_basebackup`` creates a temporary replication slot for WAL streaming and Patroni was trying to drop it because the slot name looks unknown. In order to fix it, we skip all temporary slots when querying ``pg_stat_replication_slots`` view. -- Ensure ``pg_replication_slot_advance()`` doesn't timeout (Alexander) +- Ensure ``pg_replication_slot_advance()`` doesn't timeout (Alexander Kukushkin) Patroni was using the default ``statement_timeout`` in this case and once the call failed there are very high chances that it will never recover, resulting in increased size of ``pg_wal`` and ``pg_catalog`` bloat. -- The ``/status`` wasn't updated on demote (Alexander) +- The ``/status`` wasn't updated on demote (Alexander Kukushkin) After demoting PostgreSQL the old leader updates the last LSN in DCS. Starting from ``2.1.0`` the new ``/status`` key was introduced, but the optime was still written to the ``/optime/leader``. -- Handle DCS exceptions when demoting (Alexander) +- Handle DCS exceptions when demoting (Alexander Kukushkin) While demoting the master due to failure to update the leader lock it could happen that DCS goes completely down and the ``get_cluster()`` call raises an exception. Not being handled properly it results in Postgres remaining stopped until DCS recovers. -- The ``use_unix_socket_repl`` didn't work is some cases (Alexander) +- The ``use_unix_socket_repl`` didn't work is some cases (Alexander Kukushkin) Specifically, if ``postgresql.unix_socket_directories`` is not set. In this case Patroni is supposed to use the default value from ``libpq``. -- Fix a few issues with Patroni REST API (Alexander) +- Fix a few issues with Patroni REST API (Alexander Kukushkin) The ``clusters_unlocked`` sometimes could be not defined, what resulted in exceptions in the ``GET /metrics`` endpoint. In addition to that the error handling method was assuming that the ``connect_address`` tuple always has two elements, while in fact there could be more in case of IPv6. -- Wait for newly promoted node to finish recovery before deciding to rewind (Alexander) +- Wait for newly promoted node to finish recovery before deciding to rewind (Alexander Kukushkin) It could take some time before the actual promote happens and the new timeline is created. Without waiting replicas could come to the conclusion that rewind isn't required. -- Handle missing timelines in a history file when deciding to rewind (Alexander) +- Handle missing timelines in a history file when deciding to rewind (Alexander Kukushkin) If the current replica timeline is missing in the history file on the primary the replica was falsely assuming that rewind isn't required. @@ -605,11 +605,11 @@ Version 2.1.1 The previous release added a feature of reloading REST API certificates if changed on disk. Unfortunately, the reload was happening unconditionally right after the start. -- Don't resolve cluster members when ``etcd.use_proxies`` is set (Alexander) +- Don't resolve cluster members when ``etcd.use_proxies`` is set (Alexander Kukushkin) When starting up Patroni checks the healthiness of Etcd cluster by querying the list of members. In addition to that, it also tried to resolve their hostnames, which is not necessary when working with Etcd via proxy and was causing unnecessary warnings. -- Skip rows with NULL values in the ``pg_stat_replication`` (Alexander) +- Skip rows with NULL values in the ``pg_stat_replication`` (Alexander Kukushkin) It seems that the ``pg_stat_replication`` view could contain NULL values in the ``replay_lsn``, ``flush_lsn``, or ``write_lsn`` fields even when ``state = 'streaming'``. @@ -625,11 +625,11 @@ This version adds compatibility with PostgreSQL v14, makes logical replication s Unpause WAL replay if Patroni is not in a "pause" mode itself. It could be "paused" due to the change of certain parameters like for example ``max_connections`` on the primary. -- Failover logical slots (Alexander) +- Failover logical slots (Alexander Kukushkin) Make logical replication slots survive failover/switchover on PostgreSQL v11+. The replication slot if copied from the primary to the replica with restart and later the `pg_replication_slot_advance() `__ function is used to move it forward. As a result, the slot will already exist before the failover and no events should be lost, but, there is a chance that some events could be delivered more than once. -- Implemented allowlist for Patroni REST API (Alexander) +- Implemented allowlist for Patroni REST API (Alexander Kukushkin) If configured, only IP's that matching rules would be allowed to call unsafe endpoints. In addition to that, it is possible to automatically include IP's of members of the cluster to the list. @@ -645,40 +645,40 @@ This version adds compatibility with PostgreSQL v14, makes logical replication s The endpoint exposing the same metrics as ``/patroni``. -- Reduced chattiness of Patroni logs (Alexander) +- Reduced chattiness of Patroni logs (Alexander Kukushkin) When everything goes normal, only one line will be written for every run of HA loop. **Breaking changes** -- The old ``permanent logical replication slots`` feature will no longer work with PostgreSQL v10 and older (Alexander) +- The old ``permanent logical replication slots`` feature will no longer work with PostgreSQL v10 and older (Alexander Kukushkin) The strategy of creating the logical slots after performing a promotion can't guaranty that no logical events are lost and therefore disabled. -- The ``/leader`` endpoint always returns 200 if the node holds the lock (Alexander) +- The ``/leader`` endpoint always returns 200 if the node holds the lock (Alexander Kukushkin) Promoting the standby cluster requires updating load-balancer health checks, which is not very convenient and easy to forget. To solve it, we change the behavior of the ``/leader`` health check endpoint. It will return 200 without taking into account whether the cluster is normal or the ``standby_cluster``. **Improvements in Raft support** -- Reliable support of Raft traffic encryption (Alexander) +- Reliable support of Raft traffic encryption (Alexander Kukushkin) Due to the different issues in the ``PySyncObj`` the encryption support was very unstable -- Handle DNS issues in Raft implementation (Alexander) +- Handle DNS issues in Raft implementation (Alexander Kukushkin) If ``self_addr`` and/or ``partner_addrs`` are configured using the DNS name instead of IP's the ``PySyncObj`` was effectively doing resolve only once when the object is created. It was causing problems when the same node was coming back online with a different IP. **Stability improvements** -- Compatibility with ``psycopg2-2.9+`` (Alexander) +- Compatibility with ``psycopg2-2.9+`` (Alexander Kukushkin) In ``psycopg2`` the ``autocommit = True`` is ignored in the ``with connection`` block, which breaks replication protocol connections. -- Fix excessive HA loop runs with Zookeeper (Alexander) +- Fix excessive HA loop runs with Zookeeper (Alexander Kukushkin) Update of member ZNodes was causing a chain reaction and resulted in running the HA loops multiple times in a row. @@ -690,18 +690,18 @@ This version adds compatibility with PostgreSQL v14, makes logical replication s Kerberos and password authentication are mutually exclusive. -- Fixed little issues with custom bootstrap (Alexander) +- Fixed little issues with custom bootstrap (Alexander Kukushkin) Start Postgres with ``hot_standby=off`` only when we do a PITR and restart it after PITR is done. **Bugfixes** -- Compatibility with ``kazoo-2.7+`` (Alexander) +- Compatibility with ``kazoo-2.7+`` (Alexander Kukushkin) Since Patroni is handling retries on its own, it is relying on the old behavior of ``kazoo`` that requests to a Zookeeper cluster are immediately discarded when there are no connections available. -- Explicitly request the version of Etcd v3 cluster when it is known that we are connecting via proxy (Alexander) +- Explicitly request the version of Etcd v3 cluster when it is known that we are connecting via proxy (Alexander Kukushkin) Patroni is working with Etcd v3 cluster via gPRC-gateway and it depending on the cluster version different endpoints (``/v3``, ``/v3beta``, or ``/v3alpha``) must be used. The version was resolved only together with the cluster topology, but since the latter was never done when connecting via proxy. @@ -746,11 +746,11 @@ Version 2.0.2 If Patroni notices that PostgreSQL wasn't shutdown clearly, in certain cases the crash-recovery is executed by starting Postgres in single-user mode. It could happen that the recovery failed (for example due to the lack of space on disk) but errors were swallowed. -- Added compatibility with ``python-consul2`` module (Alexander, Wilfried Roset) +- Added compatibility with ``python-consul2`` module (Alexander Kukushkin, Wilfried Roset) The good old ``python-consul`` is not maintained since a few years, therefore someone created a fork with new features and bug-fixes. -- Don't use ``bypass_api_service`` when running ``patronictl`` (Alexander) +- Don't use ``bypass_api_service`` when running ``patronictl`` (Alexander Kukushkin) When a K8s pod is running in a non-``default`` namespace it does not necessarily have enough permissions to query the ``kubernetes`` endpoint. In this case Patroni shows the warning and ignores the ``bypass_api_service`` setting. In case of ``patronictl`` the warning was a bit annoying. @@ -761,7 +761,7 @@ Version 2.0.2 **Bugfixes** -- Don't interrupt restart or promote if lost leader lock in pause (Alexander) +- Don't interrupt restart or promote if lost leader lock in pause (Alexander Kukushkin) In pause it is allowed to run postgres as primary without lock. @@ -769,7 +769,7 @@ Version 2.0.2 In order to improve handling of SSL connections and delay the handshake until thread is started Patroni overrides a few methods in the ``HTTPServer``. The ``shutdown_request()`` method was forgotten. -- Fixed issue with sleep time when using Zookeeper (Alexander) +- Fixed issue with sleep time when using Zookeeper (Alexander Kukushkin) There were chances that Patroni was sleeping up to twice longer between running HA code. @@ -777,27 +777,27 @@ Version 2.0.2 If the bootstrap failed Patroni is renaming data directory, pg_wal, and all tablespaces. After that it updates symlinks so filesystem remains consistent. The symlink creation was failing due to the ``src`` and ``dst`` arguments being swapped. -- Fixed bug in the post_bootstrap() method (Alexander) +- Fixed bug in the post_bootstrap() method (Alexander Kukushkin) If the superuser password wasn't configured Patroni was failing to call the ``post_init`` script and therefore the whole bootstrap was failing. -- Fixed an issues with pg_rewind in the standby cluster (Alexander) +- Fixed an issues with pg_rewind in the standby cluster (Alexander Kukushkin) If the superuser name is different from Postgres, the ``pg_rewind`` in the standby cluster was failing because the connection string didn't contain the database name. -- Exit only if authentication with Etcd v3 explicitly failed (Alexander) +- Exit only if authentication with Etcd v3 explicitly failed (Alexander Kukushkin) On start Patroni performs discovery of Etcd cluster topology and authenticates if it is necessarily. It could happen that one of etcd servers is not accessible, Patroni was trying to perform authentication on this server and failing instead of retrying with the next node. -- Handle case with psutil cmdline() returning empty list (Alexander) +- Handle case with psutil cmdline() returning empty list (Alexander Kukushkin) Zombie processes are still postmasters children, but they don't have cmdline() -- Treat ``PATRONI_KUBERNETES_USE_ENDPOINTS`` environment variable as boolean (Alexander) +- Treat ``PATRONI_KUBERNETES_USE_ENDPOINTS`` environment variable as boolean (Alexander Kukushkin) Not doing so was making impossible disabling ``kubernetes.use_endpoints`` via environment. -- Improve handling of concurrent endpoint update errors (Alexander) +- Improve handling of concurrent endpoint update errors (Alexander Kukushkin) Patroni will explicitly query the current endpoint object, verify that the current pod still holds the leader lock and repeat the update. @@ -821,31 +821,31 @@ Version 2.0.1 **Stability improvements** -- Changed the behavior in pause (Alexander) +- Changed the behavior in pause (Alexander Kukushkin) 1. Patroni will not call the ``bootstrap`` method if the ``PGDATA`` directory is missing/empty. 2. Patroni will not exit on sysid mismatch in pause, only log a warning. 3. The node will not try to grab the leader key in pause mode if Postgres is running not in recovery (accepting writes) but the sysid doesn't match with the initialize key. -- Apply ``master_start_timeout`` when executing crash recovery (Alexander) +- Apply ``master_start_timeout`` when executing crash recovery (Alexander Kukushkin) If Postgres crashed on the leader node, Patroni does a crash-recovery by starting Postgres in single-user mode. During the crash-recovery the leader lock is being updated. If the crash-recovery didn't finish in ``master_start_timeout`` seconds, Patroni will stop it forcefully and release the leader lock. -- Removed the ``secure`` extra from the ``urllib3`` requirements (Alexander) +- Removed the ``secure`` extra from the ``urllib3`` requirements (Alexander Kukushkin) The only reason for adding it there was the ``ipaddress`` dependency for python 2.7. **Bugfixes** -- Fixed a bug in the ``Kubernetes.update_leader()`` (Alexander) +- Fixed a bug in the ``Kubernetes.update_leader()`` (Alexander Kukushkin) An unhandled exception was preventing demoting the primary when the update of the leader object failed. -- Fixed hanging ``patronictl`` when RAFT is being used (Alexander) +- Fixed hanging ``patronictl`` when RAFT is being used (Alexander Kukushkin) When using ``patronictl`` with Patroni config, ``self_addr`` should be added to the ``partner_addrs``. -- Fixed bug in ``get_guc_value()`` (Alexander) +- Fixed bug in ``get_guc_value()`` (Alexander Kukushkin) Patroni was failing to get the value of ``restore_command`` on PostgreSQL 12, therefore fetching missing WALs for ``pg_rewind`` didn't work. @@ -861,26 +861,26 @@ This version enhances compatibility with PostgreSQL 13, adds support of multiple When promoting to ``standby_leader`` we change ``primary_conninfo``, update the role and reload Postgres. Since ``on_role_change`` and ``on_reload`` effectively duplicate each other, Patroni will call only ``on_role_change``. -- Added support for ``gssencmode`` and ``channel_binding`` connection parameters (Alexander) +- Added support for ``gssencmode`` and ``channel_binding`` connection parameters (Alexander Kukushkin) PostgreSQL 12 introduced ``gssencmode`` and 13 ``channel_binding`` connection parameters and now they can be used if defined in the ``postgresql.authentication`` section. -- Handle renaming of ``wal_keep_segments`` to ``wal_keep_size`` (Alexander) +- Handle renaming of ``wal_keep_segments`` to ``wal_keep_size`` (Alexander Kukushkin) In case of misconfiguration (``wal_keep_segments`` on 13 and ``wal_keep_size`` on older versions) Patroni will automatically adjust the configuration. -- Use ``pg_rewind`` with ``--restore-target-wal`` on 13 if possible (Alexander) +- Use ``pg_rewind`` with ``--restore-target-wal`` on 13 if possible (Alexander Kukushkin) On PostgreSQL 13 Patroni checks if ``restore_command`` is configured and tells ``pg_rewind`` to use it. **New features** -- [BETA] Implemented support of Patroni on pure RAFT (Alexander) +- [BETA] Implemented support of Patroni on pure RAFT (Alexander Kukushkin) This makes it possible to run Patroni without 3rd party dependencies, like Etcd, Consul, or Zookeeper. For HA you will have to run either three Patroni nodes or two nodes with Patroni and one node with ``patroni_raft_controller``. For more information please check the :ref:`documentation `. -- [BETA] Implemented support for Etcd v3 protocol via gPRC-gateway (Alexander) +- [BETA] Implemented support for Etcd v3 protocol via gPRC-gateway (Alexander Kukushkin) Etcd 3.0 was released more than four years ago and Etcd 3.4 has v2 disabled by default. There are also chances that v2 will be completely removed from Etcd, therefore we implemented support of Etcd v3 in Patroni. In order to start using it you have to explicitly create the ``etcd3`` section is the Patroni configuration file. @@ -896,15 +896,15 @@ This version enhances compatibility with PostgreSQL 13, adds support of multiple YAML files in the directory loaded and applied in alphabetical order. -- Advanced validation of PostgreSQL parameters (Alexander) +- Advanced validation of PostgreSQL parameters (Alexander Kukushkin) In case the specific parameter is not supported by the current PostgreSQL version or when its value is incorrect, Patroni will remove the parameter completely or try to fix the value. -- Wake up the main thread when the forced checkpoint after promote completed (Alexander) +- Wake up the main thread when the forced checkpoint after promote completed (Alexander Kukushkin) Replicas are waiting for checkpoint indication via member key of the leader in DCS. The key is normally updated only once per HA loop. Without waking the main thread up, replicas will have to wait up to ``loop_wait`` seconds longer than necessary. -- Use of ``pg_stat_wal_recevier`` view on 9.6+ (Alexander) +- Use of ``pg_stat_wal_recevier`` view on 9.6+ (Alexander Kukushkin) The view contains up-to-date values of ``primary_conninfo`` and ``primary_slot_name``, while the contents of ``recovery.conf`` could be stale. @@ -920,7 +920,7 @@ This version enhances compatibility with PostgreSQL 13, adds support of multiple It requires ``kazoo>=2.6.0``. -- Implemented ``no_params`` option for custom bootstrap method (Kostiantyn) +- Implemented ``no_params`` option for custom bootstrap method (Kostiantyn Nemchenko) It allows calling ``wal-g``, ``pgBackRest`` and other backup tools without wrapping them into shell scripts. @@ -931,61 +931,61 @@ This version enhances compatibility with PostgreSQL 13, adds support of multiple **Improved in pg_rewind support** -- Improved timeline divergence check (Alexander) +- Improved timeline divergence check (Alexander Kukushkin) We don't need to rewind when the replayed location on the replica is not ahead of the switchpoint or the end of the checkpoint record on the former primary is the same as the switchpoint. In order to get the end of the checkpoint record we use ``pg_waldump`` and parse its output. -- Try to fetch missing WAL if ``pg_rewind`` complains about it (Alexander) +- Try to fetch missing WAL if ``pg_rewind`` complains about it (Alexander Kukushkin) It could happen that the WAL segment required for ``pg_rewind`` doesn't exist in the ``pg_wal`` directory anymore and therefore ``pg_rewind`` can't find the checkpoint location before the divergence point. Starting from PostgreSQL 13 ``pg_rewind`` could use ``restore_command`` for fetching missing WALs. For older PostgreSQL versions Patroni parses the errors of a failed rewind attempt and tries to fetch the missing WAL by calling the ``restore_command`` on its own. -- Detect a new timeline in the standby cluster and trigger rewind/reinitialize if necessary (Alexander) +- Detect a new timeline in the standby cluster and trigger rewind/reinitialize if necessary (Alexander Kukushkin) The ``standby_cluster`` is decoupled from the primary cluster and therefore doesn't immediately know about leader elections and timeline switches. In order to detect the fact, the ``standby_leader`` periodically checks for new history files in ``pg_wal``. -- Shorten and beautify history log output (Alexander) +- Shorten and beautify history log output (Alexander Kukushkin) When Patroni is trying to figure out the necessity of ``pg_rewind``, it could write the content of the history file from the primary into the log. The history file is growing with every failover/switchover and eventually starts taking up too many lines, most of which are not so useful. Instead of showing the raw data, Patroni will show only 3 lines before the current replica timeline and 2 lines after. **Improvements on K8s** -- Get rid of ``kubernetes`` python module (Alexander) +- Get rid of ``kubernetes`` python module (Alexander Kukushkin) The official python kubernetes client contains a lot of auto-generated code and therefore very heavy. Patroni uses only a small fraction of K8s API endpoints and implementing support for them wasn't hard. -- Make it possible to bypass the ``kubernetes`` service (Alexander) +- Make it possible to bypass the ``kubernetes`` service (Alexander Kukushkin) When running on K8s, Patroni is usually communicating with the K8s API via the ``kubernetes`` service, the address of which is exposed in the ``KUBERNETES_SERVICE_HOST`` environment variable. Like any other service, the ``kubernetes`` service is handled by ``kube-proxy``, which in turn, depending on the configuration, is either relying on a userspace program or ``iptables`` for traffic routing. Skipping the intermediate component and connecting directly to the K8s master nodes allows us to implement a better retry strategy and mitigate risks of demoting Postgres when K8s master nodes are upgraded. -- Sync HA loops of all pods of a Patroni cluster (Alexander) +- Sync HA loops of all pods of a Patroni cluster (Alexander Kukushkin) Not doing so was increasing failure detection time from ``ttl`` to ``ttl + loop_wait``. -- Populate ``references`` and ``nodename`` in the subsets addresses on K8s (Alexander) +- Populate ``references`` and ``nodename`` in the subsets addresses on K8s (Alexander Kukushkin) Some load-balancers are relying on this information. -- Fix possible race conditions in the ``update_leader()`` (Alexander) +- Fix possible race conditions in the ``update_leader()`` (Alexander Kukushkin) The concurrent update of the leader configmap or endpoint happening outside of Patroni might cause the ``update_leader()`` call to fail. In this case Patroni rechecks that the current node is still owning the leader lock and repeats the update. -- Explicitly disallow patching non-existent config (Alexander) +- Explicitly disallow patching non-existent config (Alexander Kukushkin) For DCS other than ``kubernetes`` the PATCH call is failing with an exception due to ``cluster.config`` being ``None``, but on Kubernetes it was happily creating the config annotation and preventing writing bootstrap configuration after the bootstrap finished. -- Fix bug in ``pause`` (Alexander) +- Fix bug in ``pause`` (Alexander Kukushkin) Replicas were removing ``primary_conninfo`` and restarting Postgres when the leader key was absent, but they should do nothing. **Improvements in REST API** -- Defer TLS handshake until worker thread has started (Alexander, Ben Harris) +- Defer TLS handshake until worker thread has started (Alexander Kukushkin, Ben Harris) If the TLS handshake was done in the API thread and the client-side didn't send any data, the API thread was blocked (risking DoS). -- Check ``basic-auth`` independently from client certificate in REST API (Alexander) +- Check ``basic-auth`` independently from client certificate in REST API (Alexander Kukushkin) Previously only the client certificate was validated. Doing two checks independently is an absolutely valid use-case. @@ -993,23 +993,23 @@ This version enhances compatibility with PostgreSQL 13, adds support of multiple HAProxy was happy with a single ``CRLF``, while Consul health-check complained about broken connection and unexpected EOF. -- ``GET /cluster`` was showing stale members info for Zookeeper (Alexander) +- ``GET /cluster`` was showing stale members info for Zookeeper (Alexander Kukushkin) The endpoint was using the Patroni internal cluster view. For Patroni itself it didn't cause any issues, but when exposed to the outside world we need to show up-to-date information, especially replication lag. -- Fixed health-checks for standby cluster (Alexander) +- Fixed health-checks for standby cluster (Alexander Kukushkin) The ``GET /standby-leader`` for a master and ``GET /master`` for a ``standby_leader`` were incorrectly responding with 200. -- Implemented ``DELETE /switchover`` (Alexander) +- Implemented ``DELETE /switchover`` (Alexander Kukushkin) The REST API call deletes the scheduled switchover. -- Created ``/readiness`` and ``/liveness`` endpoints (Alexander) +- Created ``/readiness`` and ``/liveness`` endpoints (Alexander Kukushkin) They could be useful to eliminate "unhealthy" pods from subsets addresses when the K8s service is used with label selectors. -- Enhanced ``GET /replica`` and ``GET /async`` REST API health-checks (Krishna, Alexander) +- Enhanced ``GET /replica`` and ``GET /async`` REST API health-checks (Krishna Sarabu, Alexander Kukushkin) Checks now support optional keyword ``?lag=`` and will respond with 200 only if the lag is smaller than the supplied value. If relying on this feature please keep in mind that information about WAL position on the leader is updated only every ``loop_wait`` seconds! @@ -1020,34 +1020,34 @@ This version enhances compatibility with PostgreSQL 13, adds support of multiple **Improvements in patronictl** -- Don't try to call non-existing leader in ``patronictl pause`` (Alexander) +- Don't try to call non-existing leader in ``patronictl pause`` (Alexander Kukushkin) While pausing a cluster without a leader on K8s, ``patronictl`` was showing warnings that member "None" could not be accessed. -- Handle the case when member ``conn_url`` is missing (Alexander) +- Handle the case when member ``conn_url`` is missing (Alexander Kukushkin) On K8s it is possible that the pod doesn't have the necessary annotations because Patroni is not yet running. It was making ``patronictl`` to fail. -- Added ability to print ASCII cluster topology (Maxim Fedotov, Alexander) +- Added ability to print ASCII cluster topology (Maxim Fedotov, Alexander Kukushkin) It is very useful to get overview of the cluster with cascading replication. -- Implement ``patronictl flush switchover`` (Alexander) +- Implement ``patronictl flush switchover`` (Alexander Kukushkin) Before that ``patronictl flush`` only supported cancelling scheduled restarts. **Bugfixes** -- Attribute error during bootstrap of the cluster with existing PGDATA (Krishna) +- Attribute error during bootstrap of the cluster with existing PGDATA (Krishna Sarabu) When trying to create/update the ``/history`` key, Patroni was accessing the ``ClusterConfig`` object which wasn't created in DCS yet. -- Improved exception handling in Consul (Alexander) +- Improved exception handling in Consul (Alexander Kukushkin) Unhandled exception in the ``touch_member()`` method caused the whole Patroni process to crash. -- Enforce ``synchronous_commit=local`` for the ``post_init`` script (Alexander) +- Enforce ``synchronous_commit=local`` for the ``post_init`` script (Alexander Kukushkin) Patroni was already doing that when creating users (``replication``, ``rewind``), but missing it in the case of ``post_init`` was an oversight. As a result, if the script wasn't doing it internally on it's own the bootstrap in ``synchronous_mode`` wasn't able to finish. @@ -1055,19 +1055,19 @@ This version enhances compatibility with PostgreSQL 13, adds support of multiple With the default ``size=1`` some warnings were generated. -- Patroni was wrongly reporting Postgres as running (Alexander) +- Patroni was wrongly reporting Postgres as running (Alexander Kukushkin) The state wasn't updated when for example Postgres crashed due to an out-of-disk error. -- Put ``*`` into ``pgpass`` instead of missing or empty values (Alexander) +- Put ``*`` into ``pgpass`` instead of missing or empty values (Alexander Kukushkin) If for example the ``standby_cluster.port`` is not specified, the ``pgpass`` file was incorrectly generated. -- Skip physical replication slot creation on the leader node with special characters (Krishna) +- Skip physical replication slot creation on the leader node with special characters (Krishna Sarabu) Patroni appeared to be creating a dormant slot (when ``slots`` defined) for the leader node when the name contained special chars such as '-' (for e.g. "abc-us-1"). -- Avoid removing non-existent ``pg_hba.conf`` in the custom bootstrap (Krishna) +- Avoid removing non-existent ``pg_hba.conf`` in the custom bootstrap (Krishna Sarabu) Patroni was failing if ``pg_hba.conf`` happened to be located outside of the ``pgdata`` dir after custom bootstrap. @@ -1089,7 +1089,7 @@ Version 1.6.5 Use ``patroni --validate-config patroni.yaml`` in order to validate Patroni configuration. -- Possibility to configure max length of timelines history (Krishna) +- Possibility to configure max length of timelines history (Krishna Sarabu) Patroni writes the history of failovers/switchovers into the ``/history`` key in DCS. Over time the size of this key becomes big, but in most cases only the last few lines are interesting. The ``max_timelines_history`` parameter allows to specify the maximum number of timeline history items to be kept in DCS. @@ -1100,11 +1100,11 @@ Version 1.6.5 **Improvements in patronictl** -- Show member tags (Kostiantyn Nemchenko, Alexander) +- Show member tags (Kostiantyn Nemchenko, Alexander Kukushkin) Tags are configured individually for every node and there was no easy way to get an overview of them -- Improve members output (Alexander) +- Improve members output (Alexander Kukushkin) The redundant cluster name won't be shown anymore on every line, only in the table header. @@ -1125,18 +1125,18 @@ Version 1.6.5 Previously ``patronictl`` was only reporting a ``DEBUG`` message. -- Solved the problem of not initialized K8s pod breaking patronictl (Alexander) +- Solved the problem of not initialized K8s pod breaking patronictl (Alexander Kukushkin) Patroni is relying on certain pod annotations on K8s. When one of the Patroni pods is stopping or starting there is no valid annotation yet and ``patronictl`` was failing with an exception. **Stability improvements** -- Apply 1 second backoff if LIST call to K8s API server failed (Alexander) +- Apply 1 second backoff if LIST call to K8s API server failed (Alexander Kukushkin) It is mostly necessary to avoid flooding logs, but also helps to prevent starvation of the main thread. -- Retry if the ``retry-after`` HTTP header is returned by K8s API (Alexander) +- Retry if the ``retry-after`` HTTP header is returned by K8s API (Alexander Kukushkin) If the K8s API server is overwhelmed with requests it might ask to retry. @@ -1144,19 +1144,19 @@ Version 1.6.5 The ``KUBERNETES_`` environment variables are not required for PostgreSQL, yet having them exposed to the postmaster will also expose them to backends and to regular database users (using pl/perl for example). -- Clean up tablespaces on reinitialize (Krishna) +- Clean up tablespaces on reinitialize (Krishna Sarabu) During reinit, Patroni was removing only ``PGDATA`` and leaving user-defined tablespace directories. This is causing Patroni to loop in reinit. The previous workarond for the problem was implementing the :ref:`custom bootstrap ` script. -- Explicitly execute ``CHECKPOINT`` after promote happened (Alexander) +- Explicitly execute ``CHECKPOINT`` after promote happened (Alexander Kukushkin) It helps to reduce the time before the new primary is usable for ``pg_rewind``. -- Smart refresh of Etcd members (Alexander) +- Smart refresh of Etcd members (Alexander Kukushkin) In case Patroni failed to execute a request on all members of the Etcd cluster, Patroni will re-check ``A`` or ``SRV`` records for changes of IPs/hosts before retrying the next time. -- Skip missing values from ``pg_controldata`` (Feike) +- Skip missing values from ``pg_controldata`` (Feike Steenbergen) Values are missing when trying to use binaries of a version that doesn't match PGDATA. Patroni will try to start Postgres anyway, and Postgres will complain that the major version doesn't match and abort with an error. @@ -1167,15 +1167,15 @@ Version 1.6.5 Starting from a certain version of ``urllib3``, the ``cert_reqs`` must be explicitly set to ``ssl.CERT_NONE`` in order to effectively disable SSL verification. -- Avoid opening replication connection on every cycle of HA loop (Alexander) +- Avoid opening replication connection on every cycle of HA loop (Alexander Kukushkin) Regression was introduced in 1.6.4. -- Call ``on_role_change`` callback on failed primary (Alexander) +- Call ``on_role_change`` callback on failed primary (Alexander Kukushkin) In certain cases it could lead to the virtual IP remaining attached to the old primary. Regression was introduced in 1.4.5. -- Reset rewind state if postgres started after successful pg_rewind (Alexander) +- Reset rewind state if postgres started after successful pg_rewind (Alexander Kukushkin) As a result of this bug Patroni was starting up manually shut down postgres in the pause mode. @@ -1183,7 +1183,7 @@ Version 1.6.5 Patroni was indefinitely restarting replica if ``recovery_min_apply_delay`` was configured on PostgreSQL older than 12. -- PyInstaller compatibility (Alexander) +- PyInstaller compatibility (Alexander Kukushkin) PyInstaller freezes (packages) Python applications into stand-alone executables. The compatibility was broken when we switched to the ``spawn`` method instead of ``fork`` for ``multiprocessing``. @@ -1206,11 +1206,11 @@ Version 1.6.4 **Stability improvements** -- Make sure ``unix_socket_directories`` and ``stats_temp_directory`` exist (Igor) +- Make sure ``unix_socket_directories`` and ``stats_temp_directory`` exist (Igor Yanchenko) Upon the start of Patroni and Postgres make sure that ``unix_socket_directories`` and ``stats_temp_directory`` exist or try to create them. Patroni will exit if failed to create them. -- Make sure ``postgresql.pgpass`` is located in the place where Patroni has write access (Igor) +- Make sure ``postgresql.pgpass`` is located in the place where Patroni has write access (Igor Yanchenko) In case if it doesn't have a write access Patroni will exit with exception. @@ -1218,34 +1218,34 @@ Version 1.6.4 Even in case of little network problems the failing ``serfHealth`` leads to invalidation of all sessions associated with the node. Therefore, the leader key is lost much earlier than ``ttl`` which causes unwanted restarts of replicas and maybe demotion of the primary. -- Configure tcp keepalives for connections to K8s API (Alexander) +- Configure tcp keepalives for connections to K8s API (Alexander Kukushkin) In case if we get nothing from the socket after TTL seconds it can be considered dead. -- Avoid logging of passwords on user creation (Alexander) +- Avoid logging of passwords on user creation (Alexander Kukushkin) If the password is rejected or logging is configured to verbose or not configured at all it might happen that the password is written into postgres logs. In order to avoid it Patroni will change ``log_statement``, ``log_min_duration_statement``, and ``log_min_error_statement`` to some safe values before doing the attempt to create/update user. **Bugfixes** -- Use ``restore_command`` from the ``standby_cluster`` config on cascading replicas (Alexander) +- Use ``restore_command`` from the ``standby_cluster`` config on cascading replicas (Alexander Kukushkin) The ``standby_leader`` was already doing it from the beginning the feature existed. Not doing the same on replicas might prevent them from catching up with standby leader. -- Update timeline reported by the standby cluster (Alexander) +- Update timeline reported by the standby cluster (Alexander Kukushkin) In case of timeline switch the standby cluster was correctly replicating from the primary but ``patronictl`` was reporting the old timeline. -- Allow certain recovery parameters be defined in the custom_conf (Alexander) +- Allow certain recovery parameters be defined in the custom_conf (Alexander Kukushkin) When doing validation of recovery parameters on replica Patroni will skip ``archive_cleanup_command``, ``promote_trigger_file``, ``recovery_end_command``, ``recovery_min_apply_delay``, and ``restore_command`` if they are not defined in the patroni config but in files other than ``postgresql.auto.conf`` or ``postgresql.conf``. -- Improve handling of postgresql parameters with period in its name (Alexander) +- Improve handling of postgresql parameters with period in its name (Alexander Kukushkin) Such parameters could be defined by extensions where the unit is not necessarily a string. Changing the value might require a restart (for example ``pg_stat_statements.max``). -- Improve exception handling during shutdown (Alexander) +- Improve exception handling during shutdown (Alexander Kukushkin) During shutdown Patroni is trying to update its status in the DCS. If the DCS is inaccessible an exception might be raised. Lack of exception handling was preventing logger thread from stopping. @@ -1259,7 +1259,7 @@ Version 1.6.3 Bug was introduced in the `#1301 `__ -- Apply connection parameters specified in the ``postgresql.authentication`` to ``pg_basebackup`` and custom replica creation methods (Alexander) +- Apply connection parameters specified in the ``postgresql.authentication`` to ``pg_basebackup`` and custom replica creation methods (Alexander Kukushkin) They were relying on url-like connection string and therefore parameters never applied. @@ -1277,44 +1277,44 @@ Version 1.6.2 Patroni is communicating with Consul, Etcd, and Kubernetes API via the http protocol. Having a specifically crafted ``user-agent`` (example: ``Patroni/1.6.2 Python/3.6.8 Linux``) might be useful for debugging and monitoring. -- Make it possible to configure log level for exception tracebacks (Igor) +- Make it possible to configure log level for exception tracebacks (Igor Yanchenko) If you set ``log.traceback_level=DEBUG`` the tracebacks will be visible only when ``log.level=DEBUG``. The default behavior remains the same. **Stability improvements** -- Avoid importing all DCS modules when searching for the module required by the config file (Alexander) +- Avoid importing all DCS modules when searching for the module required by the config file (Alexander Kukushkin) There is no need to import modules for Etcd, Consul, and Kubernetes if we need only e.g. Zookeeper. It helps to reduce memory usage and solves the problem of having INFO messages ``Failed to import smth``. -- Removed python ``requests`` module from explicit requirements (Alexander) +- Removed python ``requests`` module from explicit requirements (Alexander Kukushkin) It wasn't used for anything critical, but causing a lot of problems when the new version of ``urllib3`` is released. -- Improve handling of ``etcd.hosts`` written as a comma-separated string instead of YAML array (Igor) +- Improve handling of ``etcd.hosts`` written as a comma-separated string instead of YAML array (Igor Yanchenko) Previously it was failing when written in format ``host1:port1, host2:port2`` (the space character after the comma). **Usability improvements** -- Don't force users to choose members from an empty list in ``patronictl`` (Igor) +- Don't force users to choose members from an empty list in ``patronictl`` (Igor Yanchenko) If the user provides a wrong cluster name, we will raise an exception rather than ask to choose a member from an empty list. -- Make the error message more helpful if the REST API cannot bind (Igor) +- Make the error message more helpful if the REST API cannot bind (Igor Yanchenko) For an inexperienced user it might be hard to figure out what is wrong from the Python stacktrace. **Bugfixes** -- Fix calculation of ``wal_buffers`` (Alexander) +- Fix calculation of ``wal_buffers`` (Alexander Kukushkin) The base unit has been changed from 8 kB blocks to bytes in PostgreSQL 11. -- Use ``passfile`` in ``primary_conninfo`` only on PostgreSQL 10+ (Alexander) +- Use ``passfile`` in ``primary_conninfo`` only on PostgreSQL 10+ (Alexander Kukushkin) On older versions there is no guarantee that ``passfile`` will work, unless the latest version of ``libpq`` is installed. @@ -1536,7 +1536,7 @@ This version adds compatibility with PostgreSQL 12, makes is possible to run pg_ You can read more about consistency mode `here `__. -- Reload Consul config on SIGHUP (Cameron Daniel, Alexander Kukushkin) +- Reload Consul config on SIGHUP (Cameron Daniel Kucera, Alexander Kukushkin) It is especially useful when somebody is changing the value of ``token``. @@ -1569,17 +1569,17 @@ Version 1.5.6 It might happen that etcd cluster is not accessible directly but via set of proxies. In this case Patroni will not perform etcd topology discovery but just round-robin via proxy hosts. Behavior is controlled by `etcd.use_proxies`. -- Changed callbacks behavior when role on the node is changed (Alexander) +- Changed callbacks behavior when role on the node is changed (Alexander Kukushkin) If the role was changed from `master` or `standby_leader` to `replica` or from `replica` to `standby_leader`, `on_restart` callback will not be called anymore in favor of `on_role_change` callback. -- Change the way how we start postgres (Alexander) +- Change the way how we start postgres (Alexander Kukushkin) Use `multiprocessing.Process` instead of executing itself and `multiprocessing.Pipe` to transmit the postmaster pid to the Patroni process. Before that we were using pipes, what was leaving postmaster process with stdin closed. **Bug fixes** -- Fix role returned by REST API for the standby leader (Alexander) +- Fix role returned by REST API for the standby leader (Alexander Kukushkin) It was incorrectly returning `replica` instead of `standby_leader` @@ -1587,11 +1587,11 @@ Version 1.5.6 Patroni doesn't have enough privileges to terminate the callback script running under `sudo` what was cancelling the new callback. If the running script could not be killed, Patroni will wait until it finishes and then run the next callback. -- Reduce lock time taken by dcs.get_cluster method (Alexander) +- Reduce lock time taken by dcs.get_cluster method (Alexander Kukushkin) Due to the lock being held DCS slowness was affecting the REST API health checks causing false positives. -- Improve cleaning of PGDATA when `pg_wal`/`pg_xlog` is a symlink (Julien) +- Improve cleaning of PGDATA when `pg_wal`/`pg_xlog` is a symlink (Julien Tachoires) In this case Patroni will explicitly remove files from the target directory. @@ -1599,7 +1599,7 @@ Version 1.5.6 It depends on being able to resolve the working directory, what will fail if Patroni is started in a directory that is later unlinked from the filesystem. -- Do not enforce ssl version when communicating with Etcd (Alexander) +- Do not enforce ssl version when communicating with Etcd (Alexander Kukushkin) For some unknown reason python3-etcd on debian and ubuntu are not based on the latest version of the package and therefore it enforces TLSv1 which is not supported by Etcd v3. We solved this problem on Patroni side. @@ -1618,25 +1618,25 @@ This version introduces the possibility of automatic reinit of the former master If the pg_rewind is disabled or can't be used, the former master could fail to start as a new replica due to diverged timelines. In this case, the only way to fix it is wiping the data directory and reinitializing. This behavior could be changed by setting `postgresql.remove_data_directory_on_diverged_timelines`. When it is set, Patroni will wipe the data directory and reinitialize the former master automatically. -- Show information about timelines in patronictl list (Alexander) +- Show information about timelines in patronictl list (Alexander Kukushkin) It helps to detect stale replicas. In addition to that, `Host` will include ':{port}' if the port value isn't default or there is more than one member running on the same host. -- Create a headless service associated with the $SCOPE-config endpoint (Alexander) +- Create a headless service associated with the $SCOPE-config endpoint (Alexander Kukushkin) The "config" endpoint keeps information about the cluster-wide Patroni and Postgres configuration, history file, and last but the most important, it holds the `initialize` key. When the Kubernetes master node is restarted or upgraded, it removes endpoints without services. The headless service will prevent it from being removed. **Bug fixes** -- Adjust the read timeout for the leader watch blocking query (Alexander) +- Adjust the read timeout for the leader watch blocking query (Alexander Kukushkin) According to the Consul documentation, the actual response timeout is increased by a small random amount of additional wait time added to the supplied maximum wait time to spread out the wake up time of any concurrent requests. It adds up to `wait / 16` additional time to the maximum duration. In our case we are adding `wait / 15` or 1 second depending on what is bigger. -- Always use replication=1 when connecting via replication protocol to the postgres (Alexander) +- Always use replication=1 when connecting via replication protocol to the postgres (Alexander Kukushkin) Starting from Postgres 10 the line in the pg_hba.conf with database=replication doesn't accept connections with the parameter replication=database. -- Don't write primary_conninfo into recovery.conf for wal-only standby cluster (Alexander) +- Don't write primary_conninfo into recovery.conf for wal-only standby cluster (Alexander Kukushkin) Despite not having neither `host` nor `port` defined in the `standby_cluster` config, Patroni was putting the `primary_conninfo` into the `recovery.conf`, which is useless and generating a lot of errors. @@ -1678,7 +1678,7 @@ This version implements flexible logging and fixes a number of bugs. They were harmless but rather annoying and sometimes scary. -- Explicitly secure rw perms for recovery.conf at creation time (Lucas) +- Explicitly secure rw perms for recovery.conf at creation time (Lucas Capistrant) We don't want anybody except patroni/postgres user reading this file, because it contains replication user and password. @@ -1709,11 +1709,11 @@ Compatibility and bugfix release. Change of `loop_wait` was causing Patroni to disconnect from zookeeper and never reconnect back. -- Fix broken compatibility with postgres 9.3 (Alexander) +- Fix broken compatibility with postgres 9.3 (Alexander Kukushkin) When opening a replication connection we should specify replication=1, because 9.3 does not understand replication='database' -- Make sure we refresh Consul session at least once per HA loop and improve handling of consul sessions exceptions (Alexander) +- Make sure we refresh Consul session at least once per HA loop and improve handling of consul sessions exceptions (Alexander Kukushkin) Restart of local consul agent invalidates all sessions related to the node. Not calling session refresh on time and not doing proper handling of session errors was causing demote of the primary. @@ -1726,7 +1726,7 @@ Compatibility and bugfix release. In order to make sure that requests are performed with an appropriate timeout, Patroni redefines create_connection method from python-kazoo module. The last release of kazoo slightly changed the way how create_connection method is called. -- Fix Patroni crash when Consul cluster loses the leader (Alexander) +- Fix Patroni crash when Consul cluster loses the leader (Alexander Kukushkin) The crash was happening due to incorrect implementation of touch_member method, it should return boolean and not raise any exceptions. @@ -1747,11 +1747,11 @@ This version implements support of permanent replication slots, adds support of **Bug fixes** -- A few bugfixes in the "standby cluster" workflow (Alexander) +- A few bugfixes in the "standby cluster" workflow (Alexander Kukushkin) Please see https://github.com/zalando/patroni/pull/823 for more details. -- Fix REST API health check when cluster management is paused and DCS is not accessible (Alexander) +- Fix REST API health check when cluster management is paused and DCS is not accessible (Alexander Kukushkin) Regression was introduced in https://github.com/zalando/patroni/commit/90cf930036a9d5249265af15d2b787ec7517cf57 @@ -1859,27 +1859,27 @@ Version 1.4.5 **Bug fixes and stability improvements** -- Fix condition for the replica start due to pg_rewind in paused state (Oleksii Kliukin) +- Fix condition for the replica start due to pg_rewind in paused state (Oleksii Kliukin) Avoid starting the replica that had already executed pg_rewind before. -- Respond 200 to the master health-check only if update_lock has been successful (Alexander) +- Respond 200 to the master health-check only if update_lock has been successful (Alexander Kukushkin) Prevent Patroni from reporting itself a master on the former (demoted) master if DCS is partitioned. -- Fix compatibility with the new consul module (Alexander) +- Fix compatibility with the new consul module (Alexander Kukushkin) Starting from v1.1.0 python-consul changed internal API and started using `list` instead of `dict` to pass query parameters. -- Catch exceptions from Patroni REST API thread during shutdown (Alexander) +- Catch exceptions from Patroni REST API thread during shutdown (Alexander Kukushkin) Those uncaught exceptions kept PostgreSQL running at shutdown. -- Do crash recovery only when Postgres runs as the master (Alexander) +- Do crash recovery only when Postgres runs as the master (Alexander Kukushkin) Require `pg_controldata` to report 'in production' or 'shutting down' or 'in crash recovery'. In all other cases no crash recovery is necessary. -- Improve handling of configuration errors (Henning Jacobs, Alexander) +- Improve handling of configuration errors (Henning Jacobs, Alexander Kukushkin) It is possible to change a lot of parameters in runtime (including `restapi.listen`) by updating Patroni config file and sending SIGHUP to Patroni process. This fix eliminates obscure exceptions from the 'restapi' thread when some of the parameters receive invalid values. @@ -1893,35 +1893,35 @@ Version 1.4.4 It didn't affect directly neither failover nor switchover, but in some rare cases it was reporting success too early, when the former leader released the lock, producing a 'Failed over to "None"' instead of 'Failed over to "desired-node"' message. -- Treat Postgres parameter names as case insensitive (Alexander) +- Treat Postgres parameter names as case insensitive (Alexander Kukushkin) Most of the Postgres parameters have snake_case names, but there are three exceptions from this rule: DateStyle, IntervalStyle and TimeZone. Postgres accepts those parameters when written in a different case (e.g. timezone = 'some/tzn'); however, Patroni was unable to find case-insensitive matches of those parameter names in pg_settings and ignored such parameters as a result. -- Abort start if attaching to running postgres and cluster not initialized (Alexander) +- Abort start if attaching to running postgres and cluster not initialized (Alexander Kukushkin) Patroni can attach itself to an already running Postgres instance. It is imperative to start running Patroni on the master node before getting to the replicas. -- Fix behavior of patronictl scaffold (Alexander) +- Fix behavior of patronictl scaffold (Alexander Kukushkin) Pass dict object to touch_member instead of json encoded string, DCS implementation will take care of encoding it. -- Don't demote master if failed to update leader key in pause (Alexander) +- Don't demote master if failed to update leader key in pause (Alexander Kukushkin) During maintenance a DCS may start failing write requests while continuing to responds to read ones. In that case, Patroni used to put the Postgres master node to a read-only mode after failing to update the leader lock in DCS. -- Sync replication slots when Patroni notices a new postmaster process (Alexander) +- Sync replication slots when Patroni notices a new postmaster process (Alexander Kukushkin) If Postgres has been restarted, Patroni has to make sure that list of replication slots matches its expectations. -- Verify sysid and sync replication slots after coming out of pause (Alexander) +- Verify sysid and sync replication slots after coming out of pause (Alexander Kukushkin) During the `maintenance` mode it may happen that data directory was completely rewritten and therefore we have to make sure that `Database system identifier` still belongs to our cluster and replication slots are in sync with Patroni expectations. -- Fix a possible failure to start not running Postgres on a data directory with postmaster lock file present (Alexander) +- Fix a possible failure to start not running Postgres on a data directory with postmaster lock file present (Alexander Kukushkin) Detect reuse of PID from the postmaster lock file. More likely to hit such problem if you run Patroni and Postgres in the docker container. -- Improve protection of DCS being accidentally wiped (Alexander) +- Improve protection of DCS being accidentally wiped (Alexander Kukushkin) Patroni has a lot of logic in place to prevent failover in such case; it can also restore all keys back; however, until this change an accidental removal of /config key was switching off pause mode for 1 cycle of HA loop. @@ -1941,7 +1941,7 @@ Version 1.4.4 If `bootstrap..keep_existing_recovery_conf` is defined and set to ``True``, Patroni will not remove the existing ``recovery.conf`` file. This is useful when bootstrapping from a backup with tools like pgBackRest that generate the appropriate `recovery.conf` for you. -- Allow options to the basebackup built-in method (Oleksii) +- Allow options to the basebackup built-in method (Oleksii Kliukin) It is now possible to supply options to the built-in basebackup method by defining the `basebackup` section in the configuration, similar to how those are defined for custom replica creation methods. The difference is in the format accepted by the `basebackup` section: since pg_basebackup accepts both `--key=value` and `--key` options, the contents of the section could be either a dictionary of key-value pairs, or a list of either one-element dictionaries or just keys (for the options that don't accept values). See :ref:`replica creation method ` section for additional examples. @@ -1963,11 +1963,11 @@ Version 1.4.3 If we have only one host in etcd configuration and exactly this host is not accessible, Patroni was starting discovery of cluster topology and never succeeding. Instead it should just switch to the next available node. -- Write content of bootstrap.pg_hba into a pg_hba.conf after custom bootstrap (Alexander) +- Write content of bootstrap.pg_hba into a pg_hba.conf after custom bootstrap (Alexander Kukushkin) Now it behaves similarly to the usual bootstrap with `initdb` -- Single user mode was waiting for user input and never finish (Alexander) +- Single user mode was waiting for user input and never finish (Alexander Kukushkin) Regression was introduced in https://github.com/zalando/patroni/pull/576 @@ -1981,28 +1981,28 @@ Version 1.4.2 Failover and switchover functions were separated in version 1.4, but `patronictl list` was still reporting `Scheduled failover` instead of `Scheduled switchover`. -- Show information about pending restarts (Alexander) +- Show information about pending restarts (Alexander Kukushkin) In order to apply some configuration changes sometimes it is necessary to restart postgres. Patroni was already giving a hint about that in the REST API and when writing node status into DCS, but there were no easy way to display it. -- Make show-config to work with cluster_name from config file (Alexander) +- Make show-config to work with cluster_name from config file (Alexander Kukushkin) It works similar to the `patronictl edit-config` **Stability improvements** -- Avoid calling pg_controldata during bootstrap (Alexander) +- Avoid calling pg_controldata during bootstrap (Alexander Kukushkin) During initdb or custom bootstrap there is a time window when pgdata is not empty but pg_controldata has not been written yet. In such case pg_controldata call was failing with error messages. -- Handle exceptions raised from psutil (Alexander) +- Handle exceptions raised from psutil (Alexander Kukushkin) cmdline is read and parsed every time when `cmdline()` method is called. It could happen that the process being examined has already disappeared, in that case `NoSuchProcess` is raised. **Kubernetes support improvements** -- Don't swallow errors from k8s API (Alexander) +- Don't swallow errors from k8s API (Alexander Kukushkin) A call to Kubernetes API could fail for a different number of reasons. In some cases such call should be retried, in some other cases we should log the error message and the exception stack trace. The change here will help debug Kubernetes permission issues. @@ -2010,7 +2010,7 @@ Version 1.4.2 Before that it was using `feature/k8s`, which became outdated. -- Add proper RBAC to run patroni on k8s (Maciej) +- Add proper RBAC to run patroni on k8s (Maciej Szulik) Add the Service account that is assigned to the pods of the cluster, the role that holds only the necessary permissions, and the rolebinding that connects the Service account and the Role. @@ -2024,7 +2024,7 @@ Version 1.4.1 patronictl failover could still work when there is leader in the cluster and it should be excluded from the list of member where it is possible to failover to. -- Make patronictl switchover compatible with the old Patroni api (Alexander) +- Make patronictl switchover compatible with the old Patroni api (Alexander Kukushkin) In case if POST /switchover REST API call has failed with status code 501 it will do it once again, but to /failover endpoint. @@ -2053,60 +2053,60 @@ In addition to using Endpoints, Patroni supports ConfigMaps. You can find more i On every iteration of HA loop Patroni needs to know recovery status and absolute wal position. From now on Patroni will run only single SELECT to get this information instead of two on the replica and three on the master. -- Remove leader key on shutdown only when we have the lock (Ants) +- Remove leader key on shutdown only when we have the lock (Ants Aasma) Unconditional removal was generating unnecessary and misleading exceptions. **Improvements in patronictl** -- Add version command to patronictl (Ants) +- Add version command to patronictl (Ants Aasma) It will show the version of installed Patroni and versions of running Patroni instances (if the cluster name is specified). -- Make optional specifying cluster_name argument for some of patronictl commands (Alexander, Ants) +- Make optional specifying cluster_name argument for some of patronictl commands (Alexander Kukushkin, Ants Aasma) It will work if patronictl is using usual Patroni configuration file with the ``scope`` defined. -- Show information about scheduled switchover and maintenance mode (Alexander) +- Show information about scheduled switchover and maintenance mode (Alexander Kukushkin) Before that it was possible to get this information only from Patroni logs or directly from DCS. -- Improve ``patronictl reinit`` (Alexander) +- Improve ``patronictl reinit`` (Alexander Kukushkin) Sometimes ``patronictl reinit`` refused to proceed when Patroni was busy with other actions, namely trying to start postgres. `patronictl` didn't provide any commands to cancel such long running actions and the only (dangerous) workarond was removing a data directory manually. The new implementation of `reinit` forcefully cancells other long-running actions before proceeding with reinit. -- Implement ``--wait`` flag in ``patronictl pause`` and ``patronictl resume`` (Alexander) +- Implement ``--wait`` flag in ``patronictl pause`` and ``patronictl resume`` (Alexander Kukushkin) It will make ``patronictl`` wait until the requested action is acknowledged by all nodes in the cluster. Such behaviour is achieved by exposing the ``pause`` flag for every node in DCS and via the REST API. -- Rename ``patronictl failover`` into ``patronictl switchover`` (Alexander) +- Rename ``patronictl failover`` into ``patronictl switchover`` (Alexander Kukushkin) The previous ``failover`` was actually only capable of doing a switchover; it refused to proceed in a cluster without the leader. -- Alter the behavior of ``patronictl failover`` (Alexander) +- Alter the behavior of ``patronictl failover`` (Alexander Kukushkin) It will work even if there is no leader, but in that case you will have to explicitly specify a node which should become the new leader. **Expose information about timeline and history** -- Expose current timeline in DCS and via API (Alexander) +- Expose current timeline in DCS and via API (Alexander Kukushkin) Store information about the current timeline for each member of the cluster. This information is accessible via the API and is stored in the DCS -- Store promotion history in the /history key in DCS (Alexander) +- Store promotion history in the /history key in DCS (Alexander Kukushkin) In addition, store the timeline history enriched with the timestamp of the corresponding promotion in the /history key in DCS and update it with each promote. **Add endpoints for getting synchronous and asynchronous replicas** -- Add new /sync and /async endpoints (Alexander, Oleksii Kliukin) +- Add new /sync and /async endpoints (Alexander Kukushkin, Oleksii Kliukin) Those endpoints (also accessible as /synchronous and /asynchronous) return 200 only for synchronous and asynchronous replicas correspondingly (exclusing those marked as `noloadbalance`). **Allow multiple hosts for Etcd** -- Add a new `hosts` parameter to Etcd configuration (Alexander) +- Add a new `hosts` parameter to Etcd configuration (Alexander Kukushkin) This parameter should contain the initial list of hosts that will be used to discover and populate the list of the running etcd cluster members. If for some reason during work this list of discovered hosts is exhausted (no available hosts from that list), Patroni will return to the initial list from the `hosts` parameter. @@ -2132,23 +2132,23 @@ Version 1.3.6 **Consul improvements** -- Make it possible to provide datacenter configuration for Consul (Vilius Okockis, Alexander) +- Make it possible to provide datacenter configuration for Consul (Vilius Okockis, Alexander Kukushkin) Before that Patroni was always communicating with datacenter of the host it runs on. -- Always send a token in X-Consul-Token http header (Alexander) +- Always send a token in X-Consul-Token http header (Alexander Kukushkin) If ``consul.token`` is defined in Patroni configuration, we will always send it in the 'X-Consul-Token' http header. python-consul module tries to be "consistent" with Consul REST API, which doesn't accept token as a query parameter for `session API `__, but it still works with 'X-Consul-Token' header. -- Adjust session TTL if supplied value is smaller than the minimum possible (Stas Fomin, Alexander) +- Adjust session TTL if supplied value is smaller than the minimum possible (Stas Fomin, Alexander Kukushkin) It could happen that the TTL provided in the Patroni configuration is smaller than the minimum one supported by Consul. In that case, Consul agent fails to create a new session. Without a session Patroni cannot create member and leader keys in the Consul KV store, resulting in an unhealthy cluster. **Other improvements** -- Define custom log format via environment variable ``PATRONI_LOGFORMAT`` (Stas) +- Define custom log format via environment variable ``PATRONI_LOGFORMAT`` (Stas Fomin) Allow disabling timestamps and other similar fields in Patroni logs if they are already added by the system logger (usually when Patroni runs as a service). @@ -2163,7 +2163,7 @@ Version 1.3.5 **Stability improvement** -- Try to run postmaster in a single-user mode if we tried and failed to start postgres (Alexander) +- Try to run postmaster in a single-user mode if we tried and failed to start postgres (Alexander Kukushkin) Usually such problem happens when node running as a master was terminated and timelines were diverged. If ``recovery.conf`` has ``restore_command`` defined, there are really high chances that postgres will abort startup and leave controldata unchanged. @@ -2171,7 +2171,7 @@ Version 1.3.5 **Consul improvements** -- Make it possible to specify health checks when creating session (Alexander) +- Make it possible to specify health checks when creating session (Alexander Kukushkin) If not specified, Consul will use "serfHealth". From one side it allows fast detection of isolated master, but from another side it makes it impossible for Patroni to tolerate short network lags. @@ -2197,23 +2197,23 @@ Version 1.3.4 possibility to specify ``scheme``, ``token``, client and ca certificates :ref:`details `. -- compatibility with python-consul-0.7.1 and above (Alexander) +- compatibility with python-consul-0.7.1 and above (Alexander Kukushkin) new python-consul module has changed signature of some methods -- "Could not take out TTL lock" message was never logged (Alexander) +- "Could not take out TTL lock" message was never logged (Alexander Kukushkin) Not a critical bug, but lack of proper logging complicates investigation in case of problems. **Quote synchronous_standby_names using quote_ident** -- When writing ``synchronous_standby_names`` into the ``postgresql.conf`` its value must be quoted (Alexander) +- When writing ``synchronous_standby_names`` into the ``postgresql.conf`` its value must be quoted (Alexander Kukushkin) If it is not quoted properly, PostgreSQL will effectively disable synchronous replication and continue to work. -**Different bugfixes around pause state, mostly related to watchdog** (Alexander) +**Different bugfixes around pause state, mostly related to watchdog** (Alexander Kukushkin) - Do not send keepalives if watchdog is not active - Avoid activating watchdog in a pause mode @@ -2227,7 +2227,7 @@ Version 1.3.3 **Bugfixes** - synchronous replication was disabled shortly after promotion even when synchronous_mode_strict was turned on (Alexander Kukushkin) -- create empty ``pg_ident.conf`` file if it is missing after restoring from the backup (Alexander) +- create empty ``pg_ident.conf`` file if it is missing after restoring from the backup (Alexander Kukushkin) - open access in ``pg_hba.conf`` to all databases, not only postgres (Franco Bellagamba) @@ -2273,7 +2273,7 @@ at the end. **Smarter pg_rewind support** -- Decide on whether to run pg_rewind by looking at the timeline differences from the current master (Alexander) +- Decide on whether to run pg_rewind by looking at the timeline differences from the current master (Alexander Kukushkin) Previously, Patroni had a fixed set of conditions to trigger pg_rewind, namely when starting a former master, when doing a switchover to the designated node for every other node in the cluster or when there is a replica with the @@ -2284,7 +2284,7 @@ at the end. **Synchronous replication mode strict** -- Enhance synchronous replication support by adding the strict mode (James Sewell, Alexander) +- Enhance synchronous replication support by adding the strict mode (James Sewell, Alexander Kukushkin) Normally, when ``synchronous_mode`` is enabled and there are no replicas attached to the master, Patroni will disable synchronous replication in order to keep the master available for writes. The ``synchronous_mode_strict`` option @@ -2295,14 +2295,14 @@ at the end. **Configuration editing with patronictl** -- Add configuration editing to patronictl (Ants Aasma, Alexander) +- Add configuration editing to patronictl (Ants Aasma, Alexander Kukushkin) Add the ability to patronictl of editing dynamic cluster configuration stored in DCS. Support either specifying the parameter/values from the command-line, invoking the $EDITOR, or applying configuration from the yaml file. **Linux watchdog support** -- Implement watchdog support for Linux (Ants) +- Implement watchdog support for Linux (Ants Aasma) Support Linux software watchdog in order to reboot the node where Patroni is not running or not responding (e.g because of the high load) The Linux software watchdog reboots the non-responsive node. It is possible to configure the watchdog @@ -2316,7 +2316,7 @@ at the end. **PostgreSQL-related minor improvements** -- Define pg_hba.conf via the Patroni configuration file or the dynamic configuration in DCS (Alexander) +- Define pg_hba.conf via the Patroni configuration file or the dynamic configuration in DCS (Alexander Kukushkin) Allow to define the contents of ``pg_hba.conf`` in the ``pg_hba`` sub-section of the ``postgresql`` section of the configuration. This simplifies managing ``pg_hba.conf`` on multiple nodes, as one needs to define it only ones in DCS @@ -2325,14 +2325,14 @@ at the end. When defined, the contents of this section will replace the current ``pg_hba.conf`` completely. Patroni ignores it if ``hba_file`` PostgreSQL parameter is set. -- Support connecting via a UNIX socket to the local PostgreSQL cluster (Alexander) +- Support connecting via a UNIX socket to the local PostgreSQL cluster (Alexander Kukushkin) Add the ``use_unix_socket`` option to the ``postgresql`` section of Patroni configuration. When set to true and the PostgreSQL ``unix_socket_directories`` option is not empty, enables Patroni to use the first value from it to connect to the local PostgreSQL cluster. If ``unix_socket_directories`` is not defined, Patroni will assume its default value and omit the ``host`` parameter in the PostgreSQL connection string altogether. -- Support change of superuser and replication credentials on reload (Alexander) +- Support change of superuser and replication credentials on reload (Alexander Kukushkin) - Support storing of configuration files outside of PostgreSQL data directory (@jouir) @@ -2341,11 +2341,11 @@ at the end. **Bug fixes and stability improvements** -- Handle EtcdEventIndexCleared and EtcdWatcherCleared exceptions (Alexander) +- Handle EtcdEventIndexCleared and EtcdWatcherCleared exceptions (Alexander Kukushkin) Faster recovery when the watch operation is ended by Etcd by avoiding useless retries. -- Remove error spinning on Etcd failure and reduce log spam (Ants) +- Remove error spinning on Etcd failure and reduce log spam (Ants Aasma) Avoid immediate retrying and emitting stack traces in the log on the second and subsequent Etcd connection failures. @@ -2353,23 +2353,23 @@ at the end. Avoid the `postmaster became multithreaded during startup` fatal error on non-English locales for PostgreSQL built with NLS. -- Extra checks when dropping the replication slot (Alexander) +- Extra checks when dropping the replication slot (Alexander Kukushkin) In some cases Patroni is prevented from dropping the replication slot by the WAL sender. - Truncate the replication slot name to 63 (NAMEDATALEN - 1) characters to comply with PostgreSQL naming rules (Nick Scott) -- Fix a race condition resulting in extra connections being opened to the PostgreSQL cluster from Patroni (Alexander) +- Fix a race condition resulting in extra connections being opened to the PostgreSQL cluster from Patroni (Alexander Kukushkin) - Release the leader key when the node restarts with an empty data directory (Alex Kerney) -- Set asynchronous executor busy when running bootstrap without a leader (Alexander) +- Set asynchronous executor busy when running bootstrap without a leader (Alexander Kukushkin) Failure to do so could have resulted in errors stating the node belonged to a different cluster, as Patroni proceeded with the normal business while being bootstrapped by a bootstrap method that doesn't require a leader to be present in the cluster. -- Improve WAL-E replica creation method (Joar Wandborg, Alexander). +- Improve WAL-E replica creation method (Joar Wandborg, Alexander Kukushkin). - Use csv.DictReader when parsing WAL-E base backup, accepting ISO dates with space-delimited date and time. - Support fetching current WAL position from the replica to estimate the amount of WAL to restore. Previously, the code used to call system information functions that were available only on the master node. @@ -2392,26 +2392,26 @@ In addition, the documentation, including these release notes, has been moved to - Do not try to update the leader position stored in the ``leader optime`` key when PostgreSQL is not 100% healthy. Demote immediately when the update of the leader key failed. (Alexander Kukushkin) -- Exclude unhealthy nodes from the list of targets to clone the new replica from. (Alexander) +- Exclude unhealthy nodes from the list of targets to clone the new replica from. (Alexander Kukushkin) -- Implement retry and timeout strategy for Consul similar to how it is done for Etcd. (Alexander) +- Implement retry and timeout strategy for Consul similar to how it is done for Etcd. (Alexander Kukushkin) -- Make ``--dcs`` and ``--config-file`` apply to all options in ``patronictl``. (Alexander) +- Make ``--dcs`` and ``--config-file`` apply to all options in ``patronictl``. (Alexander Kukushkin) -- Write all postgres parameters into postgresql.conf. (Alexander) +- Write all postgres parameters into postgresql.conf. (Alexander Kukushkin) It allows starting PostgreSQL configured by Patroni with just ``pg_ctl``. - Avoid exceptions when there are no users in the config. (Kirill Pushkin) -- Allow pausing an unhealthy cluster. Before this fix, ``patronictl`` would bail out if the node it tries to execute pause on is unhealthy. (Alexander) +- Allow pausing an unhealthy cluster. Before this fix, ``patronictl`` would bail out if the node it tries to execute pause on is unhealthy. (Alexander Kukushkin) -- Improve the leader watch functionality. (Alexander) +- Improve the leader watch functionality. (Alexander Kukushkin) Previously the replicas were always watching the leader key (sleeping until the timeout or the leader key changes). With this change, they only watch when the replica's PostgreSQL is in the ``running`` state and not when it is stopped/starting or restarting PostgreSQL. -- Avoid running into race conditions when handling SIGCHILD as a PID 1. (Alexander) +- Avoid running into race conditions when handling SIGCHILD as a PID 1. (Alexander Kukushkin) Previously a race condition could occur when running inside the Docker containers, since the same process inside Patroni both spawned new processes and handled SIGCHILD from them. This change uses fork/execs for Patroni and leaves the original PID 1 process responsible for handling signals from children. @@ -2422,24 +2422,24 @@ In addition, the documentation, including these release notes, has been moved to from WAL over the ``pg_basebackup``. This change reverts it to the original meaning of ``no_master``, namely Patroni WAL-E restore may be selected as a replication method if the master is not running. The latter is checked by examining the connection string passed to the method. In addition, it makes the retry mechanism more robust and handles other minutia. -- Implement asynchronous DNS resolver cache. (Alexander) +- Implement asynchronous DNS resolver cache. (Alexander Kukushkin) Avoid failing when DNS is temporary unavailable (for instance, due to an excessive traffic received by the node). -- Implement starting state and master start timeout. (Ants, Alexander) +- Implement starting state and master start timeout. (Ants Aasma, Alexander Kukushkin) Previously ``pg_ctl`` waited for a timeout and then happily trodded on considering PostgreSQL to be running. This caused PostgreSQL to show up in listings as running when it was actually not and caused a race condition that resulted in either a failover, or a crash recovery, or a crash recovery interrupted by failover and a missed rewind. This change adds a ``master_start_timeout`` parameter and introduces a new state for the main HA loop: ``starting``. When ``master_start_timeout`` is 0 we will failover immediately when the master crashes as soon as there is a failover candidate. Otherwise, Patroni will wait after attempting to start PostgreSQL on the master for the duration of the timeout; when it expires, it will failover if possible. Manual failover requests will be honored during the crash of the master even before the timeout expiration. Introduce the ``timeout`` parameter to the ``restart`` API endpoint and ``patronictl``. When it is set and restart takes longer than the timeout, PostgreSQL is considered unhealthy and the other nodes becomes eligible to take the leader lock. -- Fix ``pg_rewind`` behavior in a pause mode. (Ants) +- Fix ``pg_rewind`` behavior in a pause mode. (Ants Aasma) Avoid unnecessary restart in a pause mode when Patroni thinks it needs to rewind but rewind is not possible (i.e. ``pg_rewind`` is not present). Fallback to default ``libpq`` values for the ``superuser`` (default OS user) if ``superuser`` authentication is missing from the ``pg_rewind`` related Patroni configuration section. -- Serialize callback execution. Kill the previous callback of the same type when the new one is about to run. Fix the issue of spawning zombie processes when running callbacks. (Alexander) +- Serialize callback execution. Kill the previous callback of the same type when the new one is about to run. Fix the issue of spawning zombie processes when running callbacks. (Alexander Kukushkin) -- Avoid promoting a former master when the leader key is set in DCS but update to this leader key fails. (Alexander) +- Avoid promoting a former master when the leader key is set in DCS but update to this leader key fails. (Alexander Kukushkin) This avoids the issue of a current master continuing to keep its role when it is partitioned together with the minority of nodes in Etcd and other DCSs that allow "inconsistent reads". @@ -2451,21 +2451,21 @@ In addition, the documentation, including these release notes, has been moved to and sets ``PGPASSFILE`` to point to the ``.pgpass`` file containing the password. If the script fails, Patroni initialization fails as well. It is useful for adding new users or creating extensions in the new cluster. -- Implement PostgreSQL 9.6 support. (Alexander) +- Implement PostgreSQL 9.6 support. (Alexander Kukushkin) - Use ``wal_level = replica`` as a synonym for ``hot_standby``, avoiding pending_restart flag when it changes from one to another. (Alexander) + Use ``wal_level = replica`` as a synonym for ``hot_standby``, avoiding pending_restart flag when it changes from one to another. (Alexander Kukushkin) **Documentation improvements** -- Add a Patroni main `loop workflow diagram `__. (Alejandro, Alexander) +- Add a Patroni main `loop workflow diagram `__. (Alejandro Martínez, Alexander Kukushkin) - Improve README, adding the Helm chart and links to release notes. (Lauri Apple) -- Move Patroni documentation to ``Read the Docs``. The up-to-date documentation is available at https://patroni.readthedocs.io. (Oleksii) +- Move Patroni documentation to ``Read the Docs``. The up-to-date documentation is available at https://patroni.readthedocs.io. (Oleksii Kliukin) Makes the documentation easily viewable from different devices (including smartphones) and searchable. -- Move the package to the semantic versioning. (Oleksii) +- Move the package to the semantic versioning. (Oleksii Kliukin) Patroni will follow the major.minor.patch version schema to avoid releasing the new minor version on small but critical bugfixes. We will only publish the release notes for the minor version, which will include all patches. @@ -2489,28 +2489,28 @@ In addition, patronictl supports new ``pause`` and ``resume`` commands to toggle **Scheduled and conditional restarts** -- Add conditions to the restart API command (Oleksii) +- Add conditions to the restart API command (Oleksii Kliukin) This change enhances Patroni restarts by adding a couple of conditions that can be verified in order to do the restart. Among the conditions are restarting when PostgreSQL role is either a master or a replica, checking the PostgreSQL version number or restarting only when restart is necessary in order to apply configuration changes. -- Add scheduled restarts (Oleksii) +- Add scheduled restarts (Oleksii Kliukin) It is now possible to schedule a restart in the future. Only one scheduled restart per node is supported. It is possible to clear the scheduled restart if it is not needed anymore. A combination of scheduled and conditional restarts is supported, making it possible, for instance, to scheduled minor PostgreSQL upgrades in the night, restarting only the instances that are running the outdated minor version without adding postgres-specific logic to administration scripts. -- Add support for conditional and scheduled restarts to patronictl (Murat). +- Add support for conditional and scheduled restarts to patronictl (Murat Kabilov). patronictl restart supports several new options. There is also patronictl flush command to clean the scheduled actions. **Robust DCS interaction** -- Set Kazoo timeouts depending on the loop_wait (Alexander) +- Set Kazoo timeouts depending on the loop_wait (Alexander Kukushkin) Originally, ping_timeout and connect_timeout values were calculated from the negotiated session timeout. Patroni loop_wait was not taken into account. As a result, a single retry could take more time than the session timeout, forcing Patroni to release the lock and demote. This change set ping and connect timeout to half of the value of loop_wait, speeding up detection of connection issues and leaving enough time to retry the connection attempt before losing the lock. -- Update Etcd topology only after original request succeed (Alexander) +- Update Etcd topology only after original request succeed (Alexander Kukushkin) Postpone updating the Etcd topology known to the client until after the original request. When retrieving the cluster topology, implement the retry timeouts depending on the known number of nodes in the Etcd cluster. This makes our client prefer to get the results of the request to having the up-to-date list of nodes. @@ -2522,7 +2522,7 @@ In addition, patronictl supports new ``pause`` and ``resume`` commands to toggle Previously, there was no reliable way to query Patroni about PostgreSQL instances that fail to stream changes (for instance, due to connection issues). This change exposes the contents of pg_stat_replication via the /patroni endpoint. -- Add patronictl scaffold command (Oleksii) +- Add patronictl scaffold command (Oleksii Kliukin) Add a command to create cluster structure in Etcd. The cluster is created with user-specified sysid and leader, and both leader and member keys are made persistent. This command is useful to create so-called master-less configurations, where Patroni cluster consisting of only replicas replicate from the external master node that is unaware of Patroni. Subsequently, one may remove the leader key, promoting one of the Patroni nodes and replacing @@ -2538,32 +2538,32 @@ Previously, there was no reliable way to query Patroni about PostgreSQL instance **Bug fixes and code improvements** -- Make Patroni compatible with new version schema in PostgreSQL 10 and above (Feike) +- Make Patroni compatible with new version schema in PostgreSQL 10 and above (Feike Steenbergen) Make sure that Patroni understand 2-digits version numbers when doing conditional restarts based on the PostgreSQL version. -- Use pkgutil to find DCS modules (Alexander) +- Use pkgutil to find DCS modules (Alexander Kukushkin) Use the dedicated python module instead of traversing directories manually in order to find DCS modules. -- Always call on_start callback when starting Patroni (Alexander) +- Always call on_start callback when starting Patroni (Alexander Kukushkin) Previously, Patroni did not call any callbacks when attaching to the already running node with the correct role. Since callbacks are often used to route client connections that could result in the failure to register the running node in the connection routing scheme. With this fix, Patroni calls on_start callback even when attaching to the already running node. -- Do not drop active replication slots (Murat, Oleksii) +- Do not drop active replication slots (Murat Kabilov, Oleksii Kliukin) Avoid dropping active physical replication slots on master. PostgreSQL cannot drop such slots anyway. This change makes possible to run non-Patroni managed replicas/consumers on the master. -- Close Patroni connections during start of the PostgreSQL instance (Alexander) +- Close Patroni connections during start of the PostgreSQL instance (Alexander Kukushkin) Forces Patroni to close all former connections when PostgreSQL node is started. Avoids the trap of reusing former connections if postmaster was killed with SIGKILL. -- Replace invalid characters when constructing slot names from member names (Ants) +- Replace invalid characters when constructing slot names from member names (Ants Aasma) Make sure that standby names that do not comply with the slot naming rules don't cause the slot creation and standby startup to fail. Replace the dashes in the slot names with underscores and all other characters not allowed in slot names with their unicode codepoints. @@ -2594,67 +2594,67 @@ When upgrading from v0.90 or below, always upgrade all replicas before the maste Introduce `database` and `config_base_name` configuration parameters. Among others, it makes possible to run Patroni with PipelineDB and other PostgreSQL forks. -- Implement possibility to configure some Patroni configuration parameters via environment (Alexander) +- Implement possibility to configure some Patroni configuration parameters via environment (Alexander Kukushkin) Those include the scope, the node name and the namespace, as well as the secrets and makes it easier to run Patroni in a dynamic environment, i.e. Kubernetes Please, refer to the :ref:`supported environment variables ` for further details. - Update the built-in Patroni docker container to take advantage of environment-based configuration (Feike Steenbergen). -- Add Zookeeper support to Patroni docker image (Alexander) +- Add Zookeeper support to Patroni docker image (Alexander Kukushkin) -- Split the Zookeeper and Exhibitor configuration options (Alexander) +- Split the Zookeeper and Exhibitor configuration options (Alexander Kukushkin) -- Make patronictl reuse the code from Patroni to read configuration (Alexander) +- Make patronictl reuse the code from Patroni to read configuration (Alexander Kukushkin) This allows patronictl to take advantage of environment-based configuration. -- Set application name to node name in primary_conninfo (Alexander) +- Set application name to node name in primary_conninfo (Alexander Kukushkin) This simplifies identification and configuration of synchronous replication for a given node. **Stability, security and usability improvements** -- Reset sysid and do not call pg_controldata when restore of backup in progress (Alexander) +- Reset sysid and do not call pg_controldata when restore of backup in progress (Alexander Kukushkin) This change reduces the amount of noise generated by Patroni API health checks during the lengthy initialization of this node from the backup. -- Fix a bunch of pg_rewind corner-cases (Alexander) +- Fix a bunch of pg_rewind corner-cases (Alexander Kukushkin) Avoid running pg_rewind if the source cluster is not the master. In addition, avoid removing the data directory on an unsuccessful rewind, unless the new parameter *remove_data_directory_on_rewind_failure* is set to true. By default it is false. -- Remove passwords from the replication connection string in DCS (Alexander) +- Remove passwords from the replication connection string in DCS (Alexander Kukushkin) Previously, Patroni always used the replication credentials from the Postgres URL in DCS. That is now changed to take the credentials from the patroni configuration. The secrets (replication username and password) and no longer exposed in DCS. -- Fix the asynchronous machinery around the demote call (Alexander) +- Fix the asynchronous machinery around the demote call (Alexander Kukushkin) Demote now runs totally asynchronously without blocking the DCS interactions. -- Make patronictl always send the authorization header if it is configured (Alexander) +- Make patronictl always send the authorization header if it is configured (Alexander Kukushkin) This allows patronictl to issue "protected" requests, i.e. restart or reinitialize, when Patroni is configured to require authorization on those. -- Handle the SystemExit exception correctly (Alexander) +- Handle the SystemExit exception correctly (Alexander Kukushkin) Avoids the issues of Patroni not stopping properly when receiving the SIGTERM -- Sample haproxy templates for confd (Alexander) +- Sample haproxy templates for confd (Alexander Kukushkin) Generates and dynamically changes haproxy configuration from the patroni state in the DCS using confide - Improve and restructure the documentation to make it more friendly to the new users (Lauri Apple) -- API must report role=master during pg_ctl stop (Alexander) +- API must report role=master during pg_ctl stop (Alexander Kukushkin) Makes the callback calls more reliable, particularly in the cluster stop case. In addition, introduce the `pg_ctl_timeout` option to set the timeout for the start, stop and restart calls via the `pg_ctl`. -- Fix the retry logic in etcd (Alexander) +- Fix the retry logic in etcd (Alexander Kukushkin) Make retries more predictable and robust. -- Make Zookeeper code more resilient against short network hiccups (Alexander) +- Make Zookeeper code more resilient against short network hiccups (Alexander Kukushkin) Reduce the connection timeouts to make Zookeeper connection attempts more frequent. @@ -2671,17 +2671,17 @@ This releases adds support for Consul, includes a new *noloadbalance* tag, chang **New and improved tags** -- Implement *noloadbalance* tag (Alexander) +- Implement *noloadbalance* tag (Alexander Kukushkin) This tag makes Patroni always return that the replica is not available to the load balancer. -- Change the implementation of the *clonefrom* tag (Alexander) +- Change the implementation of the *clonefrom* tag (Alexander Kukushkin) Previously, a node name had to be supplied to the *clonefrom*, forcing a tagged replica to clone from the specific node. The new implementation makes *clonefrom* a boolean tag: if it is set to true, the replica becomes a candidate for other replicas to clone from it. When multiple candidates are present, the replicas picks one randomly. **Stability and security improvements** -- Numerous reliability improvements (Alexander) +- Numerous reliability improvements (Alexander Kukushkin) Removes some spurious error messages, improves the stability of the failover, addresses some corner cases with reading data from DCS, shutdown, demote and reattaching of the former leader. @@ -2693,9 +2693,9 @@ This releases adds support for Consul, includes a new *noloadbalance* tag, chang Previously, we only called *pg_rewind* if the former master had crashed. Change this to always run pg_rewind for the former master as long as pg_rewind is present in the system. This fixes the case when the master is shut down before the replicas managed to get the latest changes (i.e. during the "smart" shutdown). -- Numerous improvements to unit- and acceptance- tests, in particular, enable support for Zookeeper and Consul (Alexander). +- Numerous improvements to unit- and acceptance- tests, in particular, enable support for Zookeeper and Consul (Alexander Kukushkin). -- Make Travis CI faster and implement support for running tests against Zookeeper (Exhibitor) and Consul (Alexander) +- Make Travis CI faster and implement support for running tests against Zookeeper (Exhibitor) and Consul (Alexander Kukushkin) Both unit and acceptance tests run automatically against Etcd, Zookeeper and Consul on each commit or pull-request. @@ -2705,17 +2705,17 @@ This releases adds support for Consul, includes a new *noloadbalance* tag, chang **Configuration and control changes** -- Unify patronictl and Patroni configuration (Feike) +- Unify patronictl and Patroni configuration (Feike Steenbergen) patronictl can use the same configuration file as Patroni itself. -- Enable Patroni to read the configuration from the environment variables (Oleksii) +- Enable Patroni to read the configuration from the environment variables (Oleksii Kliukin) This simplifies generating configuration for Patroni automatically, or merging a single configuration from different sources. -- Include database system identifier in the information returned by the API (Feike) +- Include database system identifier in the information returned by the API (Feike Steenbergen) -- Implement *delete_cluster* for all available DCSs (Alexander) +- Implement *delete_cluster* for all available DCSs (Alexander Kukushkin) Enables support for DCSs other than Etcd in patronictl. @@ -2731,7 +2731,7 @@ This release adds support for *cascading replication* and simplifies Patroni man The tag *replicatefrom* allows a replica to use an arbitrary node a source, not necessary the master. The *clonefrom* does the same for the initial backup. Together, they enable Patroni to fully support cascading replication. -- Add support for running replication methods to initialize the replica even without a running replication connection (Oleksii). +- Add support for running replication methods to initialize the replica even without a running replication connection (Oleksii Kliukin). This is useful in order to create replicas from the snapshots stored on S3 or FTP. A replication method that does not require a running replication connection should supply *no_master: true* in the yaml configuration. Those scripts will still be called in order if the replication connection is present. @@ -2741,9 +2741,9 @@ This release adds support for *cascading replication* and simplifies Patroni man Failovers can be scheduled to happen at a certain time in the future, using either patronictl, or API calls. -- Add support for *dbuser* and *password* parameters in patronictl (Feike). +- Add support for *dbuser* and *password* parameters in patronictl (Feike Steenbergen). -- Add PostgreSQL version to the health check output (Feike). +- Add PostgreSQL version to the health check output (Feike Steenbergen). - Improve Zookeeper support in patronictl (Oleksandr Shulgin) @@ -2753,13 +2753,13 @@ This release adds support for *cascading replication* and simplifies Patroni man - Add a sample systems configuration script for Patroni (Jan Keirse). -- Fix the problem of Patroni ignoring the superuser name specified in the configuration file for DB connections (Alexander). +- Fix the problem of Patroni ignoring the superuser name specified in the configuration file for DB connections (Alexander Kukushkin). -- Fix the handling of CTRL-C by creating a separate session ID and process group for the postmaster launched by Patroni (Alexander). +- Fix the handling of CTRL-C by creating a separate session ID and process group for the postmaster launched by Patroni (Alexander Kukushkin). **Tests** -- Add acceptance tests with *behave* in order to check real-world scenarios of running Patroni (Alexander, Oleksii). +- Add acceptance tests with *behave* in order to check real-world scenarios of running Patroni (Alexander Kukushkin, Oleksii Kliukin). The tests can be launched manually using the *behave* command. They are also launched automatically for pull requests and after commits. From 0eea239f6b837f16f3752847f00b150fbb60ac3b Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 7 Jul 2023 09:33:30 +0200 Subject: [PATCH 18/22] Compatibility with click==8.1.4 (#2736) They somehow messed up with type hints what made pyright unhappy. To solve it we explicitly pass the Group class to the group() decorator. In addition to that bump pyright version. --- .github/workflows/tests.yaml | 2 +- patroni/ctl.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 78cce057..59a53c6e 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -173,4 +173,4 @@ jobs: - uses: jakebailey/pyright-action@v1 with: - version: 1.1.315 + version: 1.1.316 diff --git a/patroni/ctl.py b/patroni/ctl.py index 8e231466..12463aad 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -258,7 +258,7 @@ option_insecure = click.option('-k', '--insecure', is_flag=True, help='Allow con role_choice = click.Choice(['leader', 'primary', 'standby-leader', 'replica', 'standby', 'any', 'master']) -@click.group() +@click.group(cls=click.Group) @click.option('--config-file', '-c', help='Configuration file', envvar='PATRONICTL_CONFIG_FILE', default=CONFIG_FILE_PATH) @click.option('--dcs-url', '--dcs', '-d', 'dcs_url', help='The DCS connect url', envvar='DCS_URL') From c4f8e72765778daff4b2dfbb913527dbd548a88d Mon Sep 17 00:00:00 2001 From: Mark Pekala Date: Fri, 7 Jul 2023 00:57:17 -0700 Subject: [PATCH 19/22] Update .gitignore to include common venv/data patterns (#2732) Close #2731 --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 38c06f07..c3af6eb0 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,9 @@ docs/source/_templates/ #VSCode IDE .vscode/ + +# Virtual environment +venv*/ + +# Default test data directory +data/ From 4b023bc9ade1cd819603946192cd6164c1a4f65e Mon Sep 17 00:00:00 2001 From: Matt Baker <93600443+matthbakeredb@users.noreply.github.com> Date: Fri, 7 Jul 2023 10:19:41 +0100 Subject: [PATCH 20/22] Set encoding on open call in setup.py (#2727) * Set encoding on open call in setup.py If a host OS does not have a UTF-8 locale set the read() call is unable to read the utf-8 encoded README.rst file. * Remove non-ASCII characters from README.rst --- README.rst | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 18be2b49..84e67379 100644 --- a/README.rst +++ b/README.rst @@ -8,7 +8,7 @@ You can find a version of this documentation that is searchable and also easier There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation `__. -Patroni is a template for high availability (HA) PostgreSQL solutions using Python. For maximum accessibility, Patroni supports a variety of distributed configuration stores like `ZooKeeper `__, `etcd `__, `Consul `__ or `Kubernetes `__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in datacenters — or anywhere else — will hopefully find it useful. +Patroni is a template for high availability (HA) PostgreSQL solutions using Python. For maximum accessibility, Patroni supports a variety of distributed configuration stores like `ZooKeeper `__, `etcd `__, `Consul `__ or `Kubernetes `__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in datacenters - or anywhere else - will hopefully find it useful. We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely. diff --git a/setup.py b/setup.py index bff90cd4..8b14fed9 100644 --- a/setup.py +++ b/setup.py @@ -116,7 +116,7 @@ class PyTest(_Command): def read(fname): - with open(os.path.join(__location__, fname)) as fd: + with open(os.path.join(__location__, fname), encoding='utf-8') as fd: return fd.read() From 768d563fbab5f34cfb51da92f1a2cca139906179 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 7 Jul 2023 11:27:59 +0200 Subject: [PATCH 21/22] Check py files in features with flake8 (#2737) They are correctly formatted and there is no reason not to enforce it. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8b14fed9..d61eab6e 100644 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ class Flake8(_Command): yield package_directory def targets(self): - return [package for package in self.package_files()] + ['tests', 'setup.py'] + return [package for package in self.package_files()] + ['tests', 'features', 'setup.py'] def run(self): from flake8.main.cli import main From 1c36112b44b62bb668f0f23f8ad1ff9b1cff5561 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 7 Jul 2023 14:23:04 +0200 Subject: [PATCH 22/22] Reduce flakiness of citus behave tests (#2728) * Reduce flakiness of citus behave tests - make a few attempts with timeout when checking registered nodes - get rid from artificial sleep - allow check_registration() function to check secondaries These changes are useful for Quorum based failover (#2668) and future PR that enhances Citus support by registering secondaries in `pg_dist_node`. --- features/citus.feature | 17 ++++++++--------- features/steps/citus.py | 22 +++++++++++++++++----- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/features/citus.feature b/features/citus.feature index 2cf295eb..0ee08fdf 100644 --- a/features/citus.feature +++ b/features/citus.feature @@ -10,20 +10,20 @@ Feature: citus And I start postgres3 in citus group 1 Then replication works from postgres0 to postgres1 after 15 seconds Then replication works from postgres2 to postgres3 after 15 seconds - And postgres0 is registered in the postgres0 as the worker in group 0 - And postgres2 is registered in the postgres0 as the worker in group 1 + And postgres0 is registered in the postgres0 as the primary in group 0 after 5 seconds + And postgres2 is registered in the postgres0 as the primary in group 1 after 5 seconds Scenario: coordinator failover updates pg_dist_node Given I run patronictl.py failover batman --group 0 --candidate postgres1 --force Then postgres1 role is the primary after 10 seconds And replication works from postgres1 to postgres0 after 15 seconds And "sync" key in a group 0 in DCS has sync_standby=postgres0 after 15 seconds - And postgres1 is registered in the postgres2 as the worker in group 0 + And postgres1 is registered in the postgres2 as the primary in group 0 after 5 seconds When I run patronictl.py failover batman --group 0 --candidate postgres0 --force Then postgres0 role is the primary after 10 seconds And replication works from postgres0 to postgres1 after 15 seconds And "sync" key in a group 0 in DCS has sync_standby=postgres1 after 15 seconds - And postgres0 is registered in the postgres2 as the worker in group 0 + And postgres0 is registered in the postgres2 as the primary in group 0 after 5 seconds Scenario: worker switchover doesn't break client queries on the coordinator Given I create a distributed table on postgres0 @@ -33,14 +33,14 @@ Feature: citus And postgres3 role is the primary after 10 seconds And replication works from postgres3 to postgres2 after 15 seconds And "sync" key in a group 1 in DCS has sync_standby=postgres2 after 15 seconds - And postgres3 is registered in the postgres0 as the worker in group 1 + And postgres3 is registered in the postgres0 as the primary in group 1 after 5 seconds And a thread is still alive When I run patronictl.py switchover batman --group 1 --force Then I receive a response returncode 0 And postgres2 role is the primary after 10 seconds And replication works from postgres2 to postgres3 after 15 seconds And "sync" key in a group 1 in DCS has sync_standby=postgres3 after 15 seconds - And postgres2 is registered in the postgres0 as the worker in group 1 + And postgres2 is registered in the postgres0 as the primary in group 1 after 5 seconds And a thread is still alive When I stop a thread Then a distributed table on postgres0 has expected rows @@ -52,7 +52,7 @@ Feature: citus Then I receive a response returncode 0 And postgres2 role is the primary after 10 seconds And replication works from postgres2 to postgres3 after 15 seconds - And postgres2 is registered in the postgres0 as the worker in group 1 + And postgres2 is registered in the postgres0 as the primary in group 1 after 5 seconds And a thread is still alive When I stop a thread Then a distributed table on postgres0 has expected rows @@ -64,8 +64,7 @@ Feature: citus When I run patronictl.py edit-config batman --group 2 -s ttl=20 --force Then I receive a response returncode 0 And I receive a response output "+ttl: 20" - When I sleep for 2 seconds - Then postgres4 is registered in the postgres2 as the worker in group 2 + Then postgres4 is registered in the postgres2 as the primary in group 2 after 5 seconds When I shut down postgres4 Then There is a transaction in progress on postgres0 changing pg_dist_node When I run patronictl.py restart batman postgres2 --group 1 --force diff --git a/features/steps/citus.py b/features/steps/citus.py index d645a504..1af70a30 100644 --- a/features/steps/citus.py +++ b/features/steps/citus.py @@ -44,12 +44,24 @@ def start_citus(context, name, group): return context.pctl.start(name, custom_config={"citus": {"database": "postgres", "group": int(group)}}) -@step('{name1:w} is registered in the {name2:w} as the worker in group {group:d}') -def check_registration(context, name1, name2, group): +@step('{name1:w} is registered in the {name2:w} as the {role:w} in group {group:d} after {time_limit:d} seconds') +def check_registration(context, name1, name2, role, group, time_limit): + time_limit *= context.timeout_multiplier + max_time = time.time() + int(time_limit) + worker_port = int(context.pctl.query(name1, "SHOW port").fetchone()[0]) - r = context.pctl.query(name2, "SELECT nodeport FROM pg_catalog.pg_dist_node WHERE groupid = {0}".format(group)) - assert worker_port == r.fetchone()[0],\ - "Worker {0} is not registered in pg_dist_node on the coordinator {1}".format(name1, name2) + + while time.time() < max_time: + try: + cur = context.pctl.query(name2, "SELECT nodeport, noderole" + " FROM pg_catalog.pg_dist_node WHERE groupid = {0}".format(group)) + mapping = {r[0]: r[1] for r in cur} + if mapping.get(worker_port) == role: + return + except Exception: + pass + time.sleep(1) + assert False, "Node {0} is not registered in pg_dist_node on the node {1}".format(name1, name2) @step('I create a distributed table on {name:w}')