From a56b346295131c4ddd5f26bb8e2f930875de4a8a Mon Sep 17 00:00:00 2001 From: Feike Steenbergen Date: Tue, 12 May 2015 11:58:47 +0200 Subject: [PATCH] Hardened the health checks and implemented a real status page. For the postgresql helper, hardened the code to get "the" cursor of the postgresql instance. For the statuspage, a small status json is returned. To find out what status a PostgreSQL cluster is in we use the cursor (instead of the provided query() function), as the query function does some retrying etc. For the healthcheck we want to simple provide an answer to a simple query, if we have to reconnect, we are not *that* healthy anyway. Dropped catching exceptions in the do_GET block, as the HTTPServer will do that nicely for us anyway. --- helpers/postgresql.py | 17 +++++++---- helpers/statuspage.py | 66 +++++++++++++++++++++++++++++++------------ 2 files changed, 60 insertions(+), 23 deletions(-) diff --git a/helpers/postgresql.py b/helpers/postgresql.py index 4bde365c..7f4ccaad 100644 --- a/helpers/postgresql.py +++ b/helpers/postgresql.py @@ -12,6 +12,13 @@ class Postgresql: def __init__(self, config, aws_host_address=None): self.name = config["name"] self.host, self.port = config["listen"].split(":") + self.libpq_parameters = { + 'host' : aws_host_address or self.host, + 'port' : self.port, + 'fallback_application_name' : 'Governor', + 'connect_timeout' : 5, + 'options' : '-c statement_timeout=2000' + } self.data_dir = config["data_dir"] self.replication = config["replication"] self.superuser = config.get('superuser') @@ -20,15 +27,15 @@ class Postgresql: self.config = config self.cursor_holder = None - connection_host = aws_host_address or self.host - self.connection_string = "postgres://%s:%s@%s:%s/postgres" % (self.replication["username"], self.replication["password"], connection_host, self.port) + self.connection_string = "postgres://%s:%s@%s:%s/postgres" % (self.replication["username"], self.replication["password"], self.libpq_parameters['host'], self.port) self.conn = None def cursor(self): - if not self.cursor_holder: - self.conn = psycopg2.connect("postgres://%s:%s/postgres" % (self.host, self.port)) - self.conn.autocommit = True + if (self.cursor_holder is None) or self.cursor_holder.closed: + if (self.conn is None) or self.conn.closed: + self.conn = psycopg2.connect(**self.libpq_parameters) + self.conn.autocommit = True self.cursor_holder = self.conn.cursor() return self.cursor_holder diff --git a/helpers/statuspage.py b/helpers/statuspage.py index 67fdb4ab..9aae769b 100644 --- a/helpers/statuspage.py +++ b/helpers/statuspage.py @@ -2,28 +2,59 @@ # -*- coding: utf-8 -*- from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer +import json class StatusPage(BaseHTTPRequestHandler): def do_GET(self): - try: - if self.path == '/pg_master': - response = (200 if self.server.postgresql.is_leader else 503) - self.send_response(response) - elif self.path == '/pg_slave': - response = (503 if self.server.postgresql.is_leader else 200) - self.send_response(response) - elif self.path == '/pg_status': - self.send_response(200) - self.end_headers() - self.wfile.write(self.server.postgresql.status()) - else: - self.send_response(404) - except Exception, e: - self.send_response(500) - self.end_headers() - self.wfile.write(repr(e)) + if self.path == '/pg_master': + self.pg_master() + elif self.path == '/pg_slave': + self.pg_slave() + elif self.path == '/pg_status': + self.pg_status() + else: + self.send_response(404) + + def pg_master(self): + if not self.pg_is_in_recovery(): + self.send_response(200) + return + + self.send_response(503) + + def pg_slave(self): + if self.pg_is_in_recovery(): + self.send_response(200) + return + + self.send_response(503) + + def pg_is_in_recovery(self): + cursor = self.server.postgresql.cursor() + cursor.execute('SELECT pg_is_in_recovery()') + res = cursor.fetchone() + return res[0] + + def pg_status(self): + cursor = self.server.postgresql.cursor() + cursor.execute(""" + SELECT pg_is_in_recovery(), + to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'), + extract(epoch from now() - pg_last_xact_replay_timestamp()), + inet_server_addr(), + inet_server_port(), + to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ') + """) + res = cursor.fetchone() + status = {'role': ('master' if not res[0] else 'slave'), 'recovery': {'last_transaction_replayed': res[1], + 'delay': res[2]}, 'server': {'hostaddr': res[3], 'port': res[4], 'start_time': res[5]}} + + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps(status)) def getHTTPServer(postgresql, http_port=8081, listen_address='0.0.0.0'): @@ -36,7 +67,6 @@ def getHTTPServer(postgresql, http_port=8081, listen_address='0.0.0.0'): if __name__ == '__main__': import sys import logging - from BaseHTTPServer import HTTPServer logging.basicConfig(format='%(levelname)-6s %(asctime)s - %(message)s', level=logging.DEBUG) logging.debug('Starting as a standalone application')