From 80a03a4892bfa12c8a87862d0c975390da1365cd Mon Sep 17 00:00:00 2001 From: SK <78915702+sskserk@users.noreply.github.com> Date: Tue, 5 Sep 2023 07:24:17 +0200 Subject: [PATCH] Enreach some endpoints with the scope and name (#2846) - monitoring endpoints - added `name` to the `patroni`, next to the `scope` and `version` - metrics endpoint - added name to labels --- docs/rest_api.rst | 60 ++++++++++++++++++++++++++--------------------- patroni/api.py | 57 ++++++++++++++++++++++++-------------------- 2 files changed, 65 insertions(+), 52 deletions(-) diff --git a/docs/rest_api.rst b/docs/rest_api.rst index 6e97ade0..e49f6ee0 100644 --- a/docs/rest_api.rst +++ b/docs/rest_api.rst @@ -131,7 +131,8 @@ The ``GET /patroni`` is used by Patroni during the leader race. It also could be "database_system_identifier": "7268616322854375442", "patroni": { "version": "3.1.0", - "scope": "demo" + "scope": "demo", + "name": "patroni1" } } @@ -178,7 +179,8 @@ The ``GET /patroni`` is used by Patroni during the leader race. It also could be "database_system_identifier": "7268616322854375442", "patroni": { "version": "3.1.0", - "scope": "demo" + "scope": "demo", + "name": "patroni1" } } @@ -223,7 +225,8 @@ The ``GET /patroni`` is used by Patroni during the leader race. It also could be "database_system_identifier": "7268616322854375442", "patroni": { "version": "3.1.0", - "scope": "demo" + "scope": "demo", + "name": "patroni1" } } @@ -267,7 +270,8 @@ The ``GET /patroni`` is used by Patroni during the leader race. It also could be "database_system_identifier": "7268616322854375442", "patroni": { "version": "3.1.0", - "scope": "demo" + "scope": "demo", + "name": "patroni1" } } @@ -279,70 +283,70 @@ Retrieve the Patroni metrics in Prometheus format through the ``GET /metrics`` e # HELP patroni_version Patroni semver without periods. \ # TYPE patroni_version gauge - patroni_version{scope="batman"} 020103 + patroni_version{scope="batman",name="patroni1"} 020103 # HELP patroni_postgres_running Value is 1 if Postgres is running, 0 otherwise. # TYPE patroni_postgres_running gauge - patroni_postgres_running{scope="batman"} 1 + patroni_postgres_running{scope="batman",name="patroni1"} 1 # HELP patroni_postmaster_start_time Epoch seconds since Postgres started. # TYPE patroni_postmaster_start_time gauge - patroni_postmaster_start_time{scope="batman"} 1657656955.179243 + patroni_postmaster_start_time{scope="batman",name="patroni1"} 1657656955.179243 # HELP patroni_master Value is 1 if this node is the leader, 0 otherwise. # TYPE patroni_master gauge - patroni_master{scope="batman"} 1 + patroni_master{scope="batman",name="patroni1"} 1 # HELP patroni_primary Value is 1 if this node is the leader, 0 otherwise. # TYPE patroni_primary gauge - patroni_primary{scope="batman"} 1 + patroni_primary{scope="batman",name="patroni1"} 1 # HELP patroni_xlog_location Current location of the Postgres transaction log, 0 if this node is not the leader. # TYPE patroni_xlog_location counter - patroni_xlog_location{scope="batman"} 22320573386952 + patroni_xlog_location{scope="batman",name="patroni1"} 22320573386952 # HELP patroni_standby_leader Value is 1 if this node is the standby_leader, 0 otherwise. # TYPE patroni_standby_leader gauge - patroni_standby_leader{scope="batman"} 0 + patroni_standby_leader{scope="batman",name="patroni1"} 0 # HELP patroni_replica Value is 1 if this node is a replica, 0 otherwise. # TYPE patroni_replica gauge - patroni_replica{scope="batman"} 0 + patroni_replica{scope="batman",name="patroni1"} 0 # HELP patroni_sync_standby Value is 1 if this node is a sync standby replica, 0 otherwise. # TYPE patroni_sync_standby gauge - patroni_sync_standby{scope="batman"} 0 + patroni_sync_standby{scope="batman",name="patroni1"} 0 # HELP patroni_xlog_received_location Current location of the received Postgres transaction log, 0 if this node is not a replica. # TYPE patroni_xlog_received_location counter - patroni_xlog_received_location{scope="batman"} 0 + patroni_xlog_received_location{scope="batman",name="patroni1"} 0 # HELP patroni_xlog_replayed_location Current location of the replayed Postgres transaction log, 0 if this node is not a replica. # TYPE patroni_xlog_replayed_location counter - patroni_xlog_replayed_location{scope="batman"} 0 + patroni_xlog_replayed_location{scope="batman",name="patroni1"} 0 # HELP patroni_xlog_replayed_timestamp Current timestamp of the replayed Postgres transaction log, 0 if null. # TYPE patroni_xlog_replayed_timestamp gauge - patroni_xlog_replayed_timestamp{scope="batman"} 0 + patroni_xlog_replayed_timestamp{scope="batman",name="patroni1"} 0 # HELP patroni_xlog_paused Value is 1 if the Postgres xlog is paused, 0 otherwise. # TYPE patroni_xlog_paused gauge - patroni_xlog_paused{scope="batman"} 0 + patroni_xlog_paused{scope="batman",name="patroni1"} 0 # HELP patroni_postgres_streaming Value is 1 if Postgres is streaming, 0 otherwise. # TYPE patroni_postgres_streaming gauge - patroni_postgres_streaming{scope="batman"} 1 + patroni_postgres_streaming{scope="batman",name="patroni1"} 1 # HELP patroni_postgres_in_archive_recovery Value is 1 if Postgres is replicating from archive, 0 otherwise. # TYPE patroni_postgres_in_archive_recovery gauge - patroni_postgres_in_archive_recovery{scope="batman"} 0 + patroni_postgres_in_archive_recovery{scope="batman",name="patroni1"} 0 # HELP patroni_postgres_server_version Version of Postgres (if running), 0 otherwise. # TYPE patroni_postgres_server_version gauge - patroni_postgres_server_version {scope="batman"} 140004 + patroni_postgres_server_version{scope="batman",name="patroni1"} 140004 # HELP patroni_cluster_unlocked Value is 1 if the cluster is unlocked, 0 if locked. # TYPE patroni_cluster_unlocked gauge - patroni_cluster_unlocked{scope="batman"} 0 + patroni_cluster_unlocked{scope="batman",name="patroni1"} 0 # HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise. # TYPE patroni_postgres_timeline counter - patroni_failsafe_mode_is_active{scope="batman"} 0 + patroni_failsafe_mode_is_active{scope="batman",name="patroni1"} 0 # HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise. # TYPE patroni_postgres_timeline counter - patroni_postgres_timeline{scope="batman"} 24 + patroni_postgres_timeline{scope="batman",name="patroni1"} 24 # HELP patroni_dcs_last_seen Epoch timestamp when DCS was last contacted successfully by Patroni. # TYPE patroni_dcs_last_seen gauge - patroni_dcs_last_seen{scope="batman"} 1677658321 + patroni_dcs_last_seen{scope="batman",name="patroni1"} 1677658321 # HELP patroni_pending_restart Value is 1 if the node needs a restart, 0 otherwise. # TYPE patroni_pending_restart gauge - patroni_pending_restart{scope="batman"} 1 + patroni_pending_restart{scope="batman",name="patroni1"} 1 # HELP patroni_is_paused Value is 1 if auto failover is disabled, 0 otherwise. # TYPE patroni_is_paused gauge - patroni_is_paused{scope="batman"} 1 + patroni_is_paused{scope="batman",name="patroni1"} 1 Cluster status endpoints @@ -381,6 +385,7 @@ Cluster status endpoints "lag": 0 } ], + "scope": "demo", "scheduled_switchover": { "at": "2023-09-24T10:36:00+02:00", "from": "patroni1", @@ -489,8 +494,9 @@ Let's check that the node processed this configuration. First of all it should s "location": 2197818976 }, "patroni": { + "version": "1.0", "scope": "batman", - "version": "1.0" + "name": "patroni1" }, "state": "running", "role": "master", diff --git a/patroni/api.py b/patroni/api.py index b01c24d8..a7a754a3 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -198,7 +198,11 @@ class RestApiHandler(BaseHTTPRequestHandler): response['database_system_identifier'] = patroni.postgresql.sysid if patroni.postgresql.pending_restart: response['pending_restart'] = True - response['patroni'] = {'version': patroni.version, 'scope': patroni.postgresql.scope} + response['patroni'] = { + 'version': patroni.version, + 'scope': patroni.postgresql.scope, + 'name': patroni.postgresql.name + } if patroni.scheduled_restart: response['scheduled_restart'] = patroni.scheduled_restart.copy() del response['scheduled_restart']['postmaster_start_time'] @@ -449,7 +453,10 @@ class RestApiHandler(BaseHTTPRequestHandler): """ cluster = self.server.patroni.dcs.get_cluster(True) global_config = self.server.patroni.config.get_global_config(cluster) - self._write_json_response(200, cluster_as_json(cluster, global_config)) + + response = cluster_as_json(cluster, global_config) + response['scope'] = self.server.patroni.postgresql.scope + self._write_json_response(200, response) def do_GET_history(self) -> None: """Handle a ``GET`` request to ``/history`` path. @@ -526,113 +533,113 @@ class RestApiHandler(BaseHTTPRequestHandler): metrics: List[str] = [] - scope_label = '{{scope="{0}"}}'.format(patroni.postgresql.scope) + labels = f'{{scope="{patroni.postgresql.scope}",name="{patroni.postgresql.name}"}}' metrics.append("# HELP patroni_version Patroni semver without periods.") metrics.append("# TYPE patroni_version gauge") padded_semver = ''.join([x.zfill(2) for x in patroni.version.split('.')]) # 2.0.2 => 020002 - metrics.append("patroni_version{0} {1}".format(scope_label, padded_semver)) + metrics.append("patroni_version{0} {1}".format(labels, padded_semver)) metrics.append("# HELP patroni_postgres_running Value is 1 if Postgres is running, 0 otherwise.") metrics.append("# TYPE patroni_postgres_running gauge") - metrics.append("patroni_postgres_running{0} {1}".format(scope_label, int(postgres['state'] == 'running'))) + metrics.append("patroni_postgres_running{0} {1}".format(labels, int(postgres['state'] == 'running'))) metrics.append("# HELP patroni_postmaster_start_time Epoch seconds since Postgres started.") metrics.append("# TYPE patroni_postmaster_start_time gauge") postmaster_start_time = postgres.get('postmaster_start_time') postmaster_start_time = (postmaster_start_time - epoch).total_seconds() if postmaster_start_time else 0 - metrics.append("patroni_postmaster_start_time{0} {1}".format(scope_label, postmaster_start_time)) + metrics.append("patroni_postmaster_start_time{0} {1}".format(labels, postmaster_start_time)) metrics.append("# HELP patroni_master Value is 1 if this node is the leader, 0 otherwise.") metrics.append("# TYPE patroni_master gauge") - metrics.append("patroni_master{0} {1}".format(scope_label, int(postgres['role'] in ('master', 'primary')))) + metrics.append("patroni_master{0} {1}".format(labels, int(postgres['role'] in ('master', 'primary')))) metrics.append("# HELP patroni_primary Value is 1 if this node is the leader, 0 otherwise.") metrics.append("# TYPE patroni_primary gauge") - metrics.append("patroni_primary{0} {1}".format(scope_label, int(postgres['role'] in ('master', 'primary')))) + metrics.append("patroni_primary{0} {1}".format(labels, int(postgres['role'] in ('master', 'primary')))) metrics.append("# HELP patroni_xlog_location Current location of the Postgres" " transaction log, 0 if this node is not the leader.") metrics.append("# TYPE patroni_xlog_location counter") - metrics.append("patroni_xlog_location{0} {1}".format(scope_label, postgres.get('xlog', {}).get('location', 0))) + metrics.append("patroni_xlog_location{0} {1}".format(labels, postgres.get('xlog', {}).get('location', 0))) metrics.append("# HELP patroni_standby_leader Value is 1 if this node is the standby_leader, 0 otherwise.") metrics.append("# TYPE patroni_standby_leader gauge") - metrics.append("patroni_standby_leader{0} {1}".format(scope_label, int(postgres['role'] == 'standby_leader'))) + metrics.append("patroni_standby_leader{0} {1}".format(labels, int(postgres['role'] == 'standby_leader'))) metrics.append("# HELP patroni_replica Value is 1 if this node is a replica, 0 otherwise.") metrics.append("# TYPE patroni_replica gauge") - metrics.append("patroni_replica{0} {1}".format(scope_label, int(postgres['role'] == 'replica'))) + metrics.append("patroni_replica{0} {1}".format(labels, int(postgres['role'] == 'replica'))) metrics.append("# HELP patroni_sync_standby Value is 1 if this node is a sync standby replica, 0 otherwise.") metrics.append("# TYPE patroni_sync_standby gauge") - metrics.append("patroni_sync_standby{0} {1}".format(scope_label, int(postgres.get('sync_standby', False)))) + metrics.append("patroni_sync_standby{0} {1}".format(labels, int(postgres.get('sync_standby', False)))) metrics.append("# HELP patroni_xlog_received_location Current location of the received" " Postgres transaction log, 0 if this node is not a replica.") metrics.append("# TYPE patroni_xlog_received_location counter") metrics.append("patroni_xlog_received_location{0} {1}" - .format(scope_label, postgres.get('xlog', {}).get('received_location', 0))) + .format(labels, postgres.get('xlog', {}).get('received_location', 0))) metrics.append("# HELP patroni_xlog_replayed_location Current location of the replayed" " Postgres transaction log, 0 if this node is not a replica.") metrics.append("# TYPE patroni_xlog_replayed_location counter") metrics.append("patroni_xlog_replayed_location{0} {1}" - .format(scope_label, postgres.get('xlog', {}).get('replayed_location', 0))) + .format(labels, postgres.get('xlog', {}).get('replayed_location', 0))) metrics.append("# HELP patroni_xlog_replayed_timestamp Current timestamp of the replayed" " Postgres transaction log, 0 if null.") metrics.append("# TYPE patroni_xlog_replayed_timestamp gauge") replayed_timestamp = postgres.get('xlog', {}).get('replayed_timestamp') replayed_timestamp = (replayed_timestamp - epoch).total_seconds() if replayed_timestamp else 0 - metrics.append("patroni_xlog_replayed_timestamp{0} {1}".format(scope_label, replayed_timestamp)) + metrics.append("patroni_xlog_replayed_timestamp{0} {1}".format(labels, replayed_timestamp)) metrics.append("# HELP patroni_xlog_paused Value is 1 if the Postgres xlog is paused, 0 otherwise.") metrics.append("# TYPE patroni_xlog_paused gauge") metrics.append("patroni_xlog_paused{0} {1}" - .format(scope_label, int(postgres.get('xlog', {}).get('paused', False) is True))) + .format(labels, int(postgres.get('xlog', {}).get('paused', False) is True))) if postgres.get('server_version', 0) >= 90600: metrics.append("# HELP patroni_postgres_streaming Value is 1 if Postgres is streaming, 0 otherwise.") metrics.append("# TYPE patroni_postgres_streaming gauge") metrics.append("patroni_postgres_streaming{0} {1}" - .format(scope_label, int(postgres.get('replication_state') == 'streaming'))) + .format(labels, int(postgres.get('replication_state') == 'streaming'))) metrics.append("# HELP patroni_postgres_in_archive_recovery Value is 1" " if Postgres is replicating from archive, 0 otherwise.") metrics.append("# TYPE patroni_postgres_in_archive_recovery gauge") metrics.append("patroni_postgres_in_archive_recovery{0} {1}" - .format(scope_label, int(postgres.get('replication_state') == 'in archive recovery'))) + .format(labels, int(postgres.get('replication_state') == 'in archive recovery'))) metrics.append("# HELP patroni_postgres_server_version Version of Postgres (if running), 0 otherwise.") metrics.append("# TYPE patroni_postgres_server_version gauge") - metrics.append("patroni_postgres_server_version {0} {1}".format(scope_label, postgres.get('server_version', 0))) + metrics.append("patroni_postgres_server_version {0} {1}".format(labels, postgres.get('server_version', 0))) metrics.append("# HELP patroni_cluster_unlocked Value is 1 if the cluster is unlocked, 0 if locked.") metrics.append("# TYPE patroni_cluster_unlocked gauge") - metrics.append("patroni_cluster_unlocked{0} {1}".format(scope_label, int(postgres.get('cluster_unlocked', 0)))) + metrics.append("patroni_cluster_unlocked{0} {1}".format(labels, int(postgres.get('cluster_unlocked', 0)))) metrics.append("# HELP patroni_failsafe_mode_is_active Value is 1 if failsafe mode is active, 0 if inactive.") metrics.append("# TYPE patroni_failsafe_mode_is_active gauge") metrics.append("patroni_failsafe_mode_is_active{0} {1}" - .format(scope_label, int(postgres.get('failsafe_mode_is_active', 0)))) + .format(labels, int(postgres.get('failsafe_mode_is_active', 0)))) metrics.append("# HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise.") metrics.append("# TYPE patroni_postgres_timeline counter") - metrics.append("patroni_postgres_timeline{0} {1}".format(scope_label, postgres.get('timeline', 0))) + metrics.append("patroni_postgres_timeline{0} {1}".format(labels, postgres.get('timeline', 0))) metrics.append("# HELP patroni_dcs_last_seen Epoch timestamp when DCS was last contacted successfully" " by Patroni.") metrics.append("# TYPE patroni_dcs_last_seen gauge") - metrics.append("patroni_dcs_last_seen{0} {1}".format(scope_label, postgres.get('dcs_last_seen', 0))) + metrics.append("patroni_dcs_last_seen{0} {1}".format(labels, postgres.get('dcs_last_seen', 0))) metrics.append("# HELP patroni_pending_restart Value is 1 if the node needs a restart, 0 otherwise.") metrics.append("# TYPE patroni_pending_restart gauge") metrics.append("patroni_pending_restart{0} {1}" - .format(scope_label, int(patroni.postgresql.pending_restart))) + .format(labels, int(patroni.postgresql.pending_restart))) metrics.append("# HELP patroni_is_paused Value is 1 if auto failover is disabled, 0 otherwise.") metrics.append("# TYPE patroni_is_paused gauge") - metrics.append("patroni_is_paused{0} {1}".format(scope_label, int(postgres.get('pause', 0)))) + metrics.append("patroni_is_paused{0} {1}".format(labels, int(postgres.get('pause', 0)))) self.write_response(200, '\n'.join(metrics) + '\n', content_type='text/plain')