From 3b0ea900cd92762f39c2f656ae2d20e7abf20efe Mon Sep 17 00:00:00 2001 From: Christopher Winslett Date: Wed, 13 May 2015 16:37:14 -0700 Subject: [PATCH 1/4] followers without a leader based on feedback from @CyberDem0n and closes #3 --- governor.py | 2 +- helpers/ha.py | 1 + helpers/postgresql.py | 19 +++++++++++++++---- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/governor.py b/governor.py index 31081f07..c3847bd2 100755 --- a/governor.py +++ b/governor.py @@ -55,7 +55,7 @@ if postgresql.data_directory_empty(): else: time.sleep(5) else: - postgresql.write_recovery_conf({"address": "postgres://169.0.0.1:5432"}) + postgresql.follow_no_leader() postgresql.start() while True: diff --git a/helpers/ha.py b/helpers/ha.py index 80736420..bd770524 100644 --- a/helpers/ha.py +++ b/helpers/ha.py @@ -59,6 +59,7 @@ class Ha: self.state_handler.demote(self.fetch_current_leader()) return "demoting self because i am not the healthiest node" elif self.fetch_current_leader() is None: + self.state_handler.follow_no_leader() return "waiting on leader to be elected because i am not the healthiest node" else: self.state_handler.follow_the_leader(self.fetch_current_leader()) diff --git a/helpers/postgresql.py b/helpers/postgresql.py index 387ff19a..f6bbcd69 100644 --- a/helpers/postgresql.py +++ b/helpers/postgresql.py @@ -158,15 +158,18 @@ class Postgresql: f.close() def write_recovery_conf(self, leader_hash): - leader = urlparse(leader_hash["address"]) - f = open("%s/recovery.conf" % self.data_dir, "w") f.write(""" standby_mode = 'on' primary_slot_name = '%(recovery_slot)s' -primary_conninfo = 'user=%(user)s password=%(password)s host=%(hostname)s port=%(port)s sslmode=prefer sslcompression=1' recovery_target_timeline = 'latest' -""" % {"recovery_slot": self.name, "user": leader.username, "password": leader.password, "hostname": leader.hostname, "port": leader.port}) +""" % {"recovery_slot": self.name}) + if leader_hash is not None: + leader = urlparse(leader_hash["address"]) + f.write(""" +primary_conninfo = 'user=%(user)s password=%(password)s host=%(hostname)s port=%(port)s sslmode=prefer sslcompression=1' + """ % {"user": leader.username, "password": leader.password, "hostname": leader.hostname, "port": leader.port}) + if "recovery_conf" in self.config: for name, value in self.config["recovery_conf"].iteritems(): f.write("%s = '%s'\n" % (name, value)) @@ -179,6 +182,14 @@ recovery_target_timeline = 'latest' self.restart() return True + def follow_no_leader(self): + print "initing leaderless follower" + if os.system("grep primary_conninfo %(data_dir)s/recovery.conf > /dev/null" % {"data_dir": self.data_dir}) == 0: + self.write_recovery_conf(None) + if self.is_running(): + self.restart() + return True + def promote(self): return os.system("pg_ctl promote -w -D %s" % self.data_dir) == 0 From 96a889f358d82d070690465eae1838322d191e2d Mon Sep 17 00:00:00 2001 From: Christopher Winslett Date: Wed, 13 May 2015 16:56:25 -0700 Subject: [PATCH 2/4] fix logic for dead leaders returning online without checking that recovery.conf exists, the prior logic would not create the recovyer.conf, and a dead leader would return to a primary state --- helpers/postgresql.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/helpers/postgresql.py b/helpers/postgresql.py index f6bbcd69..fd93f954 100644 --- a/helpers/postgresql.py +++ b/helpers/postgresql.py @@ -92,7 +92,8 @@ class Postgresql: logger.info("Removed %s" % pid_path) command_code = os.system("postgres -D %s %s &" % (self.data_dir, self.server_options())) - time.sleep(5) + while not self.is_running(): + time.sleep(5) return command_code != 0 def stop(self): @@ -183,8 +184,7 @@ primary_conninfo = 'user=%(user)s password=%(password)s host=%(hostname)s port=% return True def follow_no_leader(self): - print "initing leaderless follower" - if os.system("grep primary_conninfo %(data_dir)s/recovery.conf > /dev/null" % {"data_dir": self.data_dir}) == 0: + if not os.path.exists("%s/recovery.conf" % self.data_dir) or os.system("grep primary_conninfo %(data_dir)s/recovery.conf &> /dev/null" % {"data_dir": self.data_dir}) == 0: self.write_recovery_conf(None) if self.is_running(): self.restart() From bec1a3c11ee464adb06aae446ccae414c100865b Mon Sep 17 00:00:00 2001 From: Christopher Winslett Date: Thu, 14 May 2015 13:02:07 -0700 Subject: [PATCH 3/4] support synchronous replication --- README.md | 23 +++++++++++++++++++++++ governor.py | 1 - helpers/postgresql.py | 4 ++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5a0fa978..7a5fa765 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,29 @@ For an example file, see `postgres0.yml`. Below is an explanation of settings: * *recovery_conf*: configuration settings written to recovery.conf when configuring follower * *parameters*: list of configuration settings for Postgres +## Replication choices + +Governor uses Postgres' streaming replication. By default, this replication is asynchronous. For more information, see the (Postgres documentation on streaming replication)[http://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION]. + +Governor's asynchronous replication configuration allows for `maximum_lag_on_failover` settings. This setting ensures replication will not occur if a follower is more than a certain number of bytes behind the follower. This setting should be increased or decreased based on business requirements. + +When asynchronous replication is not best for your use-case, investigate how Postgres's (synchronous replication)[http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION] works. Synchronous replication ensures consistency across a cluster by confirming that writes are written to a secondary before returning to the connecting client with a success. The cost of synchronous replication will be reduced throughput on writes. This throughput will be entirely based on network performance. In hosted datacenter environments (like AWS, Rackspace, or any network you do not control), synchrous replication increases the variability of write performance significantly. If followers become inaccessible from the leader, the leader will becomes effectively readonly. + +To enable a simple synchronous replication test, add the follow lines to the `parameters` section of your YAML configuration files. + +```YAML + synchronous_commit: "on" + synchronous_standby_names: "*" +``` + +When using synchronous replication, use at least a 3-Postgres data nodes to ensure write availability if one host fails. + +Choosing your replication schema is dependent on the many business decisions. Investigate both async and sync replication, as well as other HA solutions, to determine which solution is best for you. + +## Applications should not use superusers + +When connecting from an application, always use a non-superuser. Governor requires access to the database to function properly. By using a superuser from application, you can potentially use the entire connection pool, including the connections reserved for superusers with the `superuser_reserved_connections` setting. If Governor cannot access the Primary, because the connection pool is full, behavior will be undesireable. + ## Requirements on a Mac Run the following on a Mac to install requirements: diff --git a/governor.py b/governor.py index c3847bd2..ee90eaf1 100755 --- a/governor.py +++ b/governor.py @@ -40,7 +40,6 @@ if postgresql.data_directory_empty(): postgresql.initialize() etcd.take_leader(postgresql.name) postgresql.start() - postgresql.create_replication_user() else: synced_from_leader = False while not synced_from_leader: diff --git a/helpers/postgresql.py b/helpers/postgresql.py index fd93f954..b30f3c9c 100644 --- a/helpers/postgresql.py +++ b/helpers/postgresql.py @@ -56,6 +56,10 @@ class Postgresql: def initialize(self): if os.system("initdb -D %s" % self.data_dir) == 0: + # start Postgres without options to setup replication user indepedent of other system settings + os.system("pg_ctl start -w -D %s" % self.data_dir) + self.create_replication_user() + os.system("pg_ctl stop -w -m fast -D %s" % self.data_dir) self.write_pg_hba() return True From a4e40959f91388ad2306f1ebb495d682c5b28915 Mon Sep 17 00:00:00 2001 From: Christopher Winslett Date: Thu, 14 May 2015 14:18:52 -0700 Subject: [PATCH 4/4] fix linking syntax in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7a5fa765..5af483f8 100644 --- a/README.md +++ b/README.md @@ -60,11 +60,11 @@ For an example file, see `postgres0.yml`. Below is an explanation of settings: ## Replication choices -Governor uses Postgres' streaming replication. By default, this replication is asynchronous. For more information, see the (Postgres documentation on streaming replication)[http://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION]. +Governor uses Postgres' streaming replication. By default, this replication is asynchronous. For more information, see the [Postgres documentation on streaming replication](http://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION). Governor's asynchronous replication configuration allows for `maximum_lag_on_failover` settings. This setting ensures replication will not occur if a follower is more than a certain number of bytes behind the follower. This setting should be increased or decreased based on business requirements. -When asynchronous replication is not best for your use-case, investigate how Postgres's (synchronous replication)[http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION] works. Synchronous replication ensures consistency across a cluster by confirming that writes are written to a secondary before returning to the connecting client with a success. The cost of synchronous replication will be reduced throughput on writes. This throughput will be entirely based on network performance. In hosted datacenter environments (like AWS, Rackspace, or any network you do not control), synchrous replication increases the variability of write performance significantly. If followers become inaccessible from the leader, the leader will becomes effectively readonly. +When asynchronous replication is not best for your use-case, investigate how Postgres's [synchronous replication](http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION) works. Synchronous replication ensures consistency across a cluster by confirming that writes are written to a secondary before returning to the connecting client with a success. The cost of synchronous replication will be reduced throughput on writes. This throughput will be entirely based on network performance. In hosted datacenter environments (like AWS, Rackspace, or any network you do not control), synchrous replication increases the variability of write performance significantly. If followers become inaccessible from the leader, the leader will becomes effectively readonly. To enable a simple synchronous replication test, add the follow lines to the `parameters` section of your YAML configuration files.