From 1879b9c76ad2a075e90da7fe812fd434ad8d4cee Mon Sep 17 00:00:00 2001 From: Christopher Winslett Date: Mon, 4 May 2015 13:39:30 -0700 Subject: [PATCH 01/11] fix state queries against secondaries --- haproxy_status.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/haproxy_status.sh b/haproxy_status.sh index 1ddf5899..8024f59e 100755 --- a/haproxy_status.sh +++ b/haproxy_status.sh @@ -7,10 +7,10 @@ do { if [ "${response}" == " f" ] then echo "HTTP/1.1 200 OK" - echo "X-XLOG-POSITION: $(echo "SELECT pg_current_xlog_location();" | psql -t postgres 2> /dev/null | tr -d ' ' | head -n 1)" + echo "X-XLOG-POSITION: $(echo "SELECT pg_current_xlog_location();" | psql -t postgres --port $2 --host $1 2> /dev/null | tr -d ' ' | head -n 1)" else echo "HTTP/1.1 503 Service unavailable" - echo "X-XLOG-POSITION: $(echo "SELECT pg_last_xlog_replay_location();" | psql -t postgres 2> /dev/null | tr -d ' ' | head -n 1)" + echo "X-XLOG-POSITION: $(echo "SELECT pg_last_xlog_replay_location();" | psql -t postgres --port $2 --host $1 2> /dev/null | tr -d ' ' | head -n 1)" fi } | nc -l $3; done From 36c4c31e834179051ca5523f6b9083c359d64a8b Mon Sep 17 00:00:00 2001 From: Christopher Winslett Date: Sun, 10 May 2015 19:36:59 -0700 Subject: [PATCH 02/11] compare xlog positions based on bytes since 0/000000 based on feedback --- helpers/postgresql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/postgresql.py b/helpers/postgresql.py index d682bd4e..7e4d4c22 100644 --- a/helpers/postgresql.py +++ b/helpers/postgresql.py @@ -126,7 +126,7 @@ class Postgresql: member_conn = psycopg2.connect(member["address"]) member_conn.autocommit = True member_cursor = member_conn.cursor() - member_cursor.execute("SELECT '%s'::pg_lsn - pg_last_xlog_replay_location() AS bytes;" % self.xlog_position()) + member_cursor.execute("SELECT %s - (pg_last_xlog_replay_location() - '0/000000'::pg_lsn) AS bytes;" % self.xlog_position()) xlog_diff = member_cursor.fetchone()[0] logger.info([self.name, member["hostname"], xlog_diff]) if xlog_diff < 0: @@ -181,4 +181,4 @@ recovery_target_timeline = 'latest' self.query("CREATE USER \"%s\" WITH REPLICATION ENCRYPTED PASSWORD '%s';" % (self.replication["username"], self.replication["password"])) def xlog_position(self): - return self.query("SELECT pg_last_xlog_replay_location();").fetchone()[0] + return self.query("SELECT pg_last_xlog_replay_location() - '0/0000000'::pg_lsn;").fetchone()[0] From a1bc07db3374dd6336c515091b0c740a756a1bad Mon Sep 17 00:00:00 2001 From: Christopher Winslett Date: Tue, 12 May 2015 16:33:32 -0700 Subject: [PATCH 03/11] use TTLs with member listing to keep a current list of cluster members --- governor.py | 2 ++ helpers/etcd.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/governor.py b/governor.py index 57591274..31081f07 100755 --- a/governor.py +++ b/governor.py @@ -68,4 +68,6 @@ while True: if member != postgresql.name: postgresql.query("DO LANGUAGE plpgsql $$DECLARE somevar VARCHAR; BEGIN SELECT slot_name INTO somevar FROM pg_replication_slots WHERE slot_name = '%(slot)s' LIMIT 1; IF NOT FOUND THEN PERFORM pg_create_physical_replication_slot('%(slot)s'); END IF; END$$;" % {"slot": member}) + etcd.touch_member(postgresql.name, postgresql.connection_string) + time.sleep(config["loop_wait"]) diff --git a/helpers/etcd.py b/helpers/etcd.py index 88673c1e..b66e991c 100644 --- a/helpers/etcd.py +++ b/helpers/etcd.py @@ -66,7 +66,7 @@ class Etcd: raise helpers.errors.CurrentLeaderError("Etcd is not responding properly") def touch_member(self, member, connection_string): - self.put_client_path("/members/%s" % member, {"value": connection_string}) + self.put_client_path("/members/%s" % member, {"value": connection_string, "ttl": self.ttl}) def take_leader(self, value): return self.put_client_path("/leader", {"value": value, "ttl": self.ttl}) == None From 557bad37cdafaf4f5045279d1200d95ad80c2c23 Mon Sep 17 00:00:00 2001 From: Christopher Winslett Date: Tue, 12 May 2015 18:11:12 -0700 Subject: [PATCH 04/11] ensure a stale Postgres does not become leader --- helpers/etcd.py | 15 ++++++++++++--- helpers/ha.py | 7 +++++-- helpers/postgresql.py | 16 ++++++++++++++-- postgres0.yml | 1 + postgres1.yml | 1 + 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/helpers/etcd.py b/helpers/etcd.py index b66e991c..8579305c 100644 --- a/helpers/etcd.py +++ b/helpers/etcd.py @@ -79,13 +79,22 @@ class Etcd: logger.info("Could not take out TTL lock: %s" % e) return False - def update_leader(self, value): + def update_leader(self, state_handler): try: - self.put_client_path("/leader", {"value": value, "ttl": self.ttl, "prevValue": value}) + self.put_client_path("/leader", {"value": state_handler.name, "ttl": self.ttl, "prevValue": state_handler.name}) + self.put_client_path("/optime/leader", {"value": state_handler.last_operation()}) except urllib2.HTTPError: - logger.error("Error updating TTL on ETCD for primary.") + logger.error("Error updating leader lock and optime on ETCD for primary.") return False + def last_leader_operation(self): + try: + return int(self.get_client_path("/optime/leader")["node"]["value"]) + except urllib2.HTTPError as e: + if e.code == 404: + logger.error("Error updating TTL on ETCD for primary.") + return None + def leader_unlocked(self): try: self.get_client_path("/leader") diff --git a/helpers/ha.py b/helpers/ha.py index c154a2d1..d031e629 100644 --- a/helpers/ha.py +++ b/helpers/ha.py @@ -22,7 +22,10 @@ class Ha: return self.etcd.attempt_to_acquire_leader(self.state_handler.name) def update_lock(self): - return self.etcd.update_leader(self.state_handler.name) + return self.etcd.update_leader(self.state_handler) + + def update_last_leader_operation(self): + return self.etcd.update_last_leader_operation(self.state_handler.last_operation) def is_unlocked(self): return self.etcd.leader_unlocked() @@ -35,7 +38,7 @@ class Ha: def run_cycle(self): try: - if self.state_handler.is_healthy(): + if self.state_handler.is_healthy(self.etcd.last_leader_operation()): if self.is_unlocked(): if self.state_handler.is_healthiest_node(self.etcd.members()): if self.acquire_lock(): diff --git a/helpers/postgresql.py b/helpers/postgresql.py index 7e4d4c22..06ed66e0 100644 --- a/helpers/postgresql.py +++ b/helpers/postgresql.py @@ -6,7 +6,6 @@ from urlparse import urlparse logger = logging.getLogger(__name__) - class Postgresql: def __init__(self, config): @@ -111,11 +110,21 @@ class Postgresql: options += " -c \"%s=%s\"" % (setting, value) return options - def is_healthy(self): + def is_healthy(self, last_leader_operation): if not self.is_running(): logger.warning("Postgresql is not running.") return False + if self.is_leader(): + return True + + # this should only happen on initialization + if last_leader_operation is None: + return True + + if (last_leader_operation - self.xlog_position()) > self.config["maximum_lag_on_failover"]: + return False + return True def is_healthiest_node(self, members): @@ -182,3 +191,6 @@ recovery_target_timeline = 'latest' def xlog_position(self): return self.query("SELECT pg_last_xlog_replay_location() - '0/0000000'::pg_lsn;").fetchone()[0] + + def last_operation(self): + return self.query("SELECT pg_current_xlog_location() - '0/00000'::pg_lsn;").fetchone()[0] diff --git a/postgres0.yml b/postgres0.yml index 78389db6..163fd3b1 100644 --- a/postgres0.yml +++ b/postgres0.yml @@ -7,6 +7,7 @@ postgresql: name: postgresql0 listen: 127.0.0.1:5432 data_dir: data/postgresql0 + maximum_lag_on_failover: 1048576 # 1 megabyte in bytes replication: username: replicator password: rep-pass diff --git a/postgres1.yml b/postgres1.yml index 29373471..3a5bf93c 100644 --- a/postgres1.yml +++ b/postgres1.yml @@ -7,6 +7,7 @@ postgresql: name: postgresql1 listen: 127.0.0.1:5433 data_dir: data/postgresql1 + maximum_lag_on_failover: 1048576 # 1 megabyte in bytes replication: username: replicator password: rep-pass From 2c6907465249c7c9af7af7f321646959a600ad9b Mon Sep 17 00:00:00 2001 From: Christopher Winslett Date: Tue, 12 May 2015 18:29:54 -0700 Subject: [PATCH 05/11] move stale protection from is_healthy to is_healthiest to allow stale secondaries to rejoin cluster --- helpers/ha.py | 6 ++++-- helpers/postgresql.py | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/helpers/ha.py b/helpers/ha.py index d031e629..80736420 100644 --- a/helpers/ha.py +++ b/helpers/ha.py @@ -38,9 +38,9 @@ class Ha: def run_cycle(self): try: - if self.state_handler.is_healthy(self.etcd.last_leader_operation()): + if self.state_handler.is_healthy(): if self.is_unlocked(): - if self.state_handler.is_healthiest_node(self.etcd.members()): + if self.state_handler.is_healthiest_node(self.etcd): if self.acquire_lock(): if not self.state_handler.is_leader(): self.state_handler.promote() @@ -58,6 +58,8 @@ class Ha: if self.state_handler.is_leader(): self.state_handler.demote(self.fetch_current_leader()) return "demoting self because i am not the healthiest node" + elif self.fetch_current_leader() is None: + return "waiting on leader to be elected because i am not the healthiest node" else: self.state_handler.follow_the_leader(self.fetch_current_leader()) return "following a different leader because i am not the healthiest node" diff --git a/helpers/postgresql.py b/helpers/postgresql.py index 06ed66e0..387ff19a 100644 --- a/helpers/postgresql.py +++ b/helpers/postgresql.py @@ -110,7 +110,7 @@ class Postgresql: options += " -c \"%s=%s\"" % (setting, value) return options - def is_healthy(self, last_leader_operation): + def is_healthy(self): if not self.is_running(): logger.warning("Postgresql is not running.") return False @@ -118,17 +118,17 @@ class Postgresql: if self.is_leader(): return True - # this should only happen on initialization - if last_leader_operation is None: - return True - - if (last_leader_operation - self.xlog_position()) > self.config["maximum_lag_on_failover"]: - return False - return True - def is_healthiest_node(self, members): - for member in members: + def is_healthiest_node(self, state_store): + # this should only happen on initialization + if state_store.last_leader_operation() is None: + return True + + if (state_store.last_leader_operation() - self.xlog_position()) > self.config["maximum_lag_on_failover"]: + return False + + for member in state_store.members(): if member["hostname"] == self.name: continue try: From f8c4993f16b52c2881bd0b8332c8f6767ec999f7 Mon Sep 17 00:00:00 2001 From: Christopher Winslett Date: Tue, 12 May 2015 18:47:40 -0700 Subject: [PATCH 06/11] add description of settings for yaml file --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 9e825948..5a0fa978 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,29 @@ We provide a haproxy configuration, which will give your application a single en For a diagram of the high availability decision loop, see the included a PDF: [postgres-ha.pdf](https://github.com/compose/template-etcd-based-postgres-ha/blob/master/postgres-ha.pdf) +## YAML Configuration + +For an example file, see `postgres0.yml`. Below is an explanation of settings: + +* *loop_wait*: the number of seconds the loop will sleep + +* *etcd* + * *scope*: the relative path used on etcd's http api for this deployment, thus you can run multiple HA deployments from a single etcd + * *ttl*: the TTL to acquire the leader lock. Think of it as the length of time before automatic failover process is initiated. + * *host*: the host:port for the etcd endpoint + +* *postgresql* + * *name*: the name of the Postgres host, must be unique for the cluster + * *listen*: ip address + port that Postgres listening. Must be accessible from other nodes in the cluster if using streaming replication. + * *data_dir*: file path to initialize and store Postgres data files + * *maximum_lag_on_failover*: the maximum bytes a follower may lag before it is not eligible become leader + * *replication* + * *username*: replication username, user will be created during initialization + * *password*: replication password, user will be created during initialization + * *network*: network setting for replication in pg_hba.conf + * *recovery_conf*: configuration settings written to recovery.conf when configuring follower + * *parameters*: list of configuration settings for Postgres + ## Requirements on a Mac Run the following on a Mac to install requirements: From e6e19359df4df3a9eb27fa817133c36983b2d5af Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 13 May 2015 18:04:27 +0200 Subject: [PATCH 07/11] Bugfix, write state_handler.name into leader key --- helpers/etcd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/etcd.py b/helpers/etcd.py index cecb8b78..4513435f 100644 --- a/helpers/etcd.py +++ b/helpers/etcd.py @@ -126,7 +126,7 @@ class Etcd: return ret def update_leader(self, state_handler): - ret = self.put_client_path('/leader', value=value, ttl=self.ttl, prevValue=state_handler.name) + ret = self.put_client_path('/leader', value=state_handler.name, ttl=self.ttl, prevValue=state_handler.name) ret and self.put_client_path('/optime/leader', value=state_handler.last_operation()) return ret From 4e8a0873a7bcb346992cad6512d865f278373ce9 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 13 May 2015 18:05:42 +0200 Subject: [PATCH 08/11] Handle sigchld, so we can run governor and postgres inside docker safely --- governor.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/governor.py b/governor.py index 39e19685..86404280 100755 --- a/governor.py +++ b/governor.py @@ -16,6 +16,17 @@ def sigterm_handler(signo, stack_frame): sys.exit() +# handle SIGCHILD, since we are the equivalent of the INIT process +def sigchld_handler(signo, stack_frame): + try: + while True: + ret = os.waitpid(-1, os.WNOHANG) + if ret == (0, 0): + break + except OSError: + pass + + class Governor: def __init__(self, config): @@ -78,4 +89,5 @@ def main(): if __name__ == '__main__': logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO) signal.signal(signal.SIGTERM, sigterm_handler) + signal.signal(signal.SIGCHLD, sigchld_handler) main() From 9487aec842449f9a39eb115ae937205fe371207c Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 15 May 2015 11:23:02 +0200 Subject: [PATCH 09/11] Bugfix: get_client_path might get into infinite loop when response code is not 200 --- helpers/etcd.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helpers/etcd.py b/helpers/etcd.py index 4513435f..cbe44507 100644 --- a/helpers/etcd.py +++ b/helpers/etcd.py @@ -45,6 +45,8 @@ class Etcd: time.sleep(3) elif ex: raise ex + else: + break return response.json(), response.status_code From 8e5beb58837a5b620bba1c7fe12044bfaf93a134 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 15 May 2015 11:25:26 +0200 Subject: [PATCH 10/11] Small optimization --- helpers/postgresql.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/helpers/postgresql.py b/helpers/postgresql.py index afc397ca..79d290c3 100644 --- a/helpers/postgresql.py +++ b/helpers/postgresql.py @@ -144,11 +144,7 @@ class Postgresql: return True def is_healthiest_node(self, last_leader_operation, members): - # this should only happen on initialization - if last_leader_operation is None: - return True - - if last_leader_operation - self.xlog_position() > self.config['maximum_lag_on_failover']: + if last_leader_operation and last_leader_operation - self.xlog_position() > self.config['maximum_lag_on_failover']: return False for member in members: From 274bc14bfb6cfb50b608dd396eee7a9b7370cc33 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 15 May 2015 11:38:50 +0200 Subject: [PATCH 11/11] Do not check maximum_lag_on_failover if it is not defined in config file --- helpers/postgresql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/postgresql.py b/helpers/postgresql.py index 79d290c3..a061a52c 100644 --- a/helpers/postgresql.py +++ b/helpers/postgresql.py @@ -144,7 +144,7 @@ class Postgresql: return True def is_healthiest_node(self, last_leader_operation, members): - if last_leader_operation and last_leader_operation - self.xlog_position() > self.config['maximum_lag_on_failover']: + if (last_leader_operation or 0) - self.xlog_position() > self.config.get('maximum_lag_on_failover', 0): return False for member in members: