Hardened the health checks and implemented a real status page.

For the postgresql helper, hardened the code to get "the" cursor of the postgresql instance.
For the statuspage, a small status json is returned.
To find out what status a PostgreSQL cluster is in we use the cursor (instead of the provided query() function), as the query function does some retrying etc. For the healthcheck we want to simple provide an answer to a simple query, if we have to reconnect, we are not *that* healthy anyway.

Dropped catching exceptions in the do_GET block, as the HTTPServer will do that nicely for us anyway.

helpers/postgresql.py

@@ -12,6 +12,13 @@ class Postgresql:
     def __init__(self, config, aws_host_address=None):
         self.name = config["name"]
         self.host, self.port = config["listen"].split(":")
+        self.libpq_parameters = {
+            'host' : aws_host_address or self.host,
+            'port' : self.port,
+            'fallback_application_name' : 'Governor',
+            'connect_timeout' : 5,
+            'options' : '-c statement_timeout=2000'
+        }
         self.data_dir = config["data_dir"]
         self.replication = config["replication"]
         self.superuser = config.get('superuser')
@@ -20,15 +27,15 @@ class Postgresql:
         self.config = config
 
         self.cursor_holder = None
-        connection_host = aws_host_address or self.host
-        self.connection_string = "postgres://%s:%s@%s:%s/postgres" % (self.replication["username"], self.replication["password"], connection_host, self.port)
+        self.connection_string = "postgres://%s:%s@%s:%s/postgres" % (self.replication["username"], self.replication["password"], self.libpq_parameters['host'], self.port)
 
         self.conn = None
 
     def cursor(self):
-        if not self.cursor_holder:
-            self.conn = psycopg2.connect("postgres://%s:%s/postgres" % (self.host, self.port))
-            self.conn.autocommit = True
+        if (self.cursor_holder is None) or self.cursor_holder.closed:
+            if (self.conn is None) or self.conn.closed:
+                self.conn = psycopg2.connect(**self.libpq_parameters)
+                self.conn.autocommit = True
             self.cursor_holder = self.conn.cursor()
 
         return self.cursor_holder

helpers/statuspage.py

@@ -2,28 +2,59 @@
 # -*- coding: utf-8 -*-
 
 from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+import json
 
 
 class StatusPage(BaseHTTPRequestHandler):
 
     def do_GET(self):
-        try:
-            if self.path == '/pg_master':
-                response = (200 if self.server.postgresql.is_leader else 503)
-                self.send_response(response)
-            elif self.path == '/pg_slave':
-                response = (503 if self.server.postgresql.is_leader else 200)
-                self.send_response(response)
-            elif self.path == '/pg_status':
-                self.send_response(200)
-                self.end_headers()
-                self.wfile.write(self.server.postgresql.status())
-            else:
-                self.send_response(404)
-        except Exception, e:
-            self.send_response(500)
-            self.end_headers()
-            self.wfile.write(repr(e))
+        if self.path == '/pg_master':
+            self.pg_master()
+        elif self.path == '/pg_slave':
+            self.pg_slave()
+        elif self.path == '/pg_status':
+            self.pg_status()
+        else:
+            self.send_response(404)
+
+    def pg_master(self):
+        if not self.pg_is_in_recovery():
+            self.send_response(200)
+            return
+
+        self.send_response(503)
+
+    def pg_slave(self):
+        if self.pg_is_in_recovery():
+            self.send_response(200)
+            return
+
+        self.send_response(503)
+
+    def pg_is_in_recovery(self):
+        cursor = self.server.postgresql.cursor()
+        cursor.execute('SELECT pg_is_in_recovery()')
+        res = cursor.fetchone()
+        return res[0]
+
+    def pg_status(self):
+        cursor = self.server.postgresql.cursor()
+        cursor.execute("""
+            SELECT pg_is_in_recovery(),
+                   to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
+                   extract(epoch from now() - pg_last_xact_replay_timestamp()),
+                   inet_server_addr(),
+                   inet_server_port(),
+                   to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ')
+                    """)
+        res = cursor.fetchone()
+        status = {'role': ('master' if not res[0] else 'slave'), 'recovery': {'last_transaction_replayed': res[1],
+                  'delay': res[2]}, 'server': {'hostaddr': res[3], 'port': res[4], 'start_time': res[5]}}
+
+        self.send_response(200)
+        self.send_header('Content-Type', 'application/json')
+        self.end_headers()
+        self.wfile.write(json.dumps(status))
 
 
 def getHTTPServer(postgresql, http_port=8081, listen_address='0.0.0.0'):
@@ -36,7 +67,6 @@ def getHTTPServer(postgresql, http_port=8081, listen_address='0.0.0.0'):
 if __name__ == '__main__':
     import sys
     import logging
-    from BaseHTTPServer import HTTPServer
 
     logging.basicConfig(format='%(levelname)-6s %(asctime)s - %(message)s', level=logging.DEBUG)
     logging.debug('Starting as a standalone application')