From ef0b3c2296ed5150de3be1827ab593afde560446 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 23 Aug 2016 11:46:16 +0200 Subject: [PATCH 1/5] Bring all configs to the new format (#265) The v1.0 has been released more than one month ago and the new version is coming. It doesn't make a lot of sense to keep configuration files in the old format anymore. In addition to that I've also commented out all the lines enabling and configuring "archiving" to avoid incidents like here: https://github.com/zalando/patroni/issues/264 --- postgres0.yml | 14 +++-- postgres1.yml | 152 ++++++++++++++++++++----------------------------- postgres2.yml | 154 +++++++++++++++++++++----------------------------- 3 files changed, 133 insertions(+), 187 deletions(-) diff --git a/postgres0.yml b/postgres0.yml index 499501bb..7933eb99 100644 --- a/postgres0.yml +++ b/postgres0.yml @@ -4,10 +4,12 @@ name: postgresql0 restapi: listen: 127.0.0.1:8008 + connect_address: 127.0.0.1:8008 +# certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem +# keyfile: /etc/ssl/private/ssl-cert-snakeoil.key # authentication: # username: username # password: password - connect_address: 127.0.0.1:8008 etcd: host: 127.0.0.1:4001 @@ -30,11 +32,11 @@ bootstrap: # max_wal_senders: 5 # max_replication_slots: 5 # wal_log_hints: "on" - archive_mode: "on" - archive_timeout: 1800s - archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f - recovery_conf: - restore_command: cp ../wal_archive/%f %p +# archive_mode: "on" +# archive_timeout: 1800s +# archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f +# recovery_conf: +# restore_command: cp ../wal_archive/%f %p # some desired options for 'initdb' initdb: # Note: It needs to be a list (some options need values, others are switches) diff --git a/postgres1.yml b/postgres1.yml index 5d2b58ee..35187a2b 100644 --- a/postgres1.yml +++ b/postgres1.yml @@ -1,105 +1,77 @@ -ttl: &ttl 30 -loop_wait: &loop_wait 10 -scope: &scope batman +scope: batman +#namespace: /service/ +name: postgresql1 + restapi: listen: 127.0.0.1:8009 connect_address: 127.0.0.1:8009 -# auth: 'username:password' # certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem # keyfile: /etc/ssl/private/ssl-cert-snakeoil.key +# authentication: +# username: username +# password: password + etcd: - scope: *scope - ttl: *ttl host: 127.0.0.1:4001 - #discovery_srv: my-etcd.domain -#consul: -# scope: *scope -# ttl: *ttl -# host: 127.0.0.1:8500 -#zookeeper: -# scope: *scope -# session_timeout: *ttl -# reconnect_timeout: *loop_wait -# hosts: -# - 127.0.0.1:2181 -# - 127.0.0.2:2181 -#exhibitor: -# poll_interval: 300 -# port: 8181 -# hosts: -# - host1 -# - host2 -# - host3 + +bootstrap: + # this section will be written into Etcd:///config after initializing new cluster + # and all other cluster members will use it as a `global configuration` + dcs: + ttl: 30 + loop_wait: 10 + retry_timeout: 10 + maximum_lag_on_failover: 1048576 + postgresql: + use_pg_rewind: true +# use_slots: true + parameters: +# wal_level: hot_standby +# hot_standby: "on" +# wal_keep_segments: 8 +# max_wal_senders: 5 +# max_replication_slots: 5 +# wal_log_hints: "on" +# archive_mode: "on" +# archive_timeout: 1800s +# archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f +# recovery_conf: +# restore_command: cp ../wal_archive/%f %p + + # some desired options for 'initdb' + initdb: # Note: It needs to be a list (some options need values, others are switches) + - encoding: UTF8 + - data-checksums + + pg_hba: # Add following lines to pg_hba.conf after running 'initdb' + - host replication replicator 127.0.0.1/32 md5 + - host all all 0.0.0.0/0 md5 +# - hostssl all all 0.0.0.0/0 md5 + + # Some additional users users which needs to be created after initializing new cluster + users: + admin: + password: admin + options: + - createrole + - createdb + postgresql: - name: postgresql1 - scope: *scope listen: 127.0.0.1:5433 connect_address: 127.0.0.1:5433 data_dir: data/postgresql1 - maximum_lag_on_failover: 1048576 # 1 megabyte in bytes - use_slots: True +# bin_dir: pgpass: /tmp/pgpass1 - initdb: ## We allow the following options to be passed on to initdb - # - auth: authmethod - # - auth-host: authmethod - # - auth-local: authmethod - - encoding: UTF8 - # - data-checksums # When pg_rewind is needed on 9.3, this needs to be enabled - # - locale: locale - # - lc-collate: locale - # - lc-ctype: locale - # - lc-messages: locale - # - lc-monetary: locale - # - lc-numeric: locale - # - lc-time: locale - # - text-search-config: CFG - # - xlogdir: directory - # - debug - # - noclean - pg_rewind: - username: postgres - password: zalando - pg_hba: - - host replication replicator 127.0.0.1/32 md5 - - host all all 0.0.0.0/0 md5 - # - hostssl all all 0.0.0.0/0 md5 - replication: - username: replicator - password: rep-pass - superuser: - username: postgres - password: zalando - admin: - username: admin - password: admin -# commented-out example for wal-e provisioning - create_replica_method: - - basebackup -# - wal_e -# commented-out example for wal-e provisioning - #wal_e: - #command: /patroni/scripts/wale_restore.py - #env_dir: /home/postgres/etc/wal-e.d/env - #threshold_megabytes: 10240 - #threshold_backup_size_percentage: 30 - #retries: 2 - #use_iam: 1 - #recovery_conf: - #restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1 - recovery_conf: - restore_command: cp ../wal_archive/%f %p + authentication: + replication: + username: replicator + password: rep-pass + superuser: + username: postgres + password: zalando parameters: - archive_mode: "on" - wal_level: hot_standby - archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f - max_wal_senders: 10 - wal_keep_segments: 8 - archive_timeout: 1800s - max_replication_slots: 10 - hot_standby: "on" - wal_log_hints: "on" unix_socket_directories: '.' tags: - nofailover: False - noloadbalance: False - clonefrom: False + nofailover: false + noloadbalance: false + clonefrom: false diff --git a/postgres2.yml b/postgres2.yml index 48c3b55d..3ea34b9d 100644 --- a/postgres2.yml +++ b/postgres2.yml @@ -1,106 +1,78 @@ -ttl: &ttl 30 -loop_wait: &loop_wait 10 -scope: &scope batman +scope: batman +#namespace: /service/ +name: postgresql2 + restapi: listen: 127.0.0.1:8010 connect_address: 127.0.0.1:8010 - auth: 'username:password' # certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem # keyfile: /etc/ssl/private/ssl-cert-snakeoil.key + authentication: + username: username + password: password + etcd: - scope: *scope - ttl: *ttl host: 127.0.0.1:4001 - #discovery_srv: my-etcd.domain -#consul: -# scope: *scope -# ttl: *ttl -# host: 127.0.0.1:8500 -#zookeeper: -# scope: *scope -# session_timeout: *ttl -# reconnect_timeout: *loop_wait -# hosts: -# - 127.0.0.1:2181 -# - 127.0.0.2:2181 -#exhibitor: -# poll_interval: 300 -# port: 8181 -# hosts: -# - host1 -# - host2 -# - host3 + +bootstrap: + # this section will be written into Etcd:///config after initializing new cluster + # and all other cluster members will use it as a `global configuration` + dcs: + ttl: 30 + loop_wait: 10 + retry_timeout: 10 + maximum_lag_on_failover: 1048576 + postgresql: + use_pg_rewind: true +# use_slots: true + parameters: +# wal_level: hot_standby +# hot_standby: "on" +# wal_keep_segments: 8 +# max_wal_senders: 5 +# max_replication_slots: 5 +# wal_log_hints: "on" +# archive_mode: "on" +# archive_timeout: 1800s +# archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f +# recovery_conf: +# restore_command: cp ../wal_archive/%f %p + + # some desired options for 'initdb' + initdb: # Note: It needs to be a list (some options need values, others are switches) + - encoding: UTF8 + - data-checksums + + pg_hba: # Add following lines to pg_hba.conf after running 'initdb' + - host replication replicator 127.0.0.1/32 md5 + - host all all 0.0.0.0/0 md5 +# - hostssl all all 0.0.0.0/0 md5 + + # Some additional users users which needs to be created after initializing new cluster + users: + admin: + password: admin + options: + - createrole + - createdb + postgresql: - name: postgresql2 - scope: *scope listen: 127.0.0.1:5434 connect_address: 127.0.0.1:5434 data_dir: data/postgresql2 - maximum_lag_on_failover: 1048576 # 1 megabyte in bytes - use_slots: True +# bin_dir: pgpass: /tmp/pgpass2 - initdb: ## We allow the following options to be passed on to initdb - # - auth: authmethod - # - auth-host: authmethod - # - auth-local: authmethod - - encoding: UTF8 - # - data-checksums # When pg_rewind is needed on 9.3, this needs to be enabled - # - locale: locale - # - lc-collate: locale - # - lc-ctype: locale - # - lc-messages: locale - # - lc-monetary: locale - # - lc-numeric: locale - # - lc-time: locale - # - text-search-config: CFG - # - xlogdir: directory - # - debug - # - noclean - pg_rewind: - username: postgres - password: zalando - pg_hba: - - host replication replicator 127.0.0.1/32 md5 - - host all all 0.0.0.0/0 md5 - # - hostssl all all 0.0.0.0/0 md5 - replication: - username: replicator - password: rep-pass - superuser: - username: postgres - password: zalando - admin: - username: admin - password: admin -# commented-out example for wal-e provisioning - create_replica_method: - - basebackup -# - wal_e -# commented-out example for wal-e provisioning - #wal_e: - #command: /patroni/scripts/wale_restore.py - #env_dir: /home/postgres/etc/wal-e.d/env - #threshold_megabytes: 10240 - #threshold_backup_size_percentage: 30 - #retries: 2 - #use_iam: 1 - #recovery_conf: - #restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1 - recovery_conf: - restore_command: cp ../wal_archive/%f %p + authentication: + replication: + username: replicator + password: rep-pass + superuser: + username: postgres + password: zalando parameters: - archive_mode: "on" - wal_level: hot_standby - archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f - max_wal_senders: 10 - wal_keep_segments: 8 - archive_timeout: 1800s - max_replication_slots: 10 - hot_standby: "on" - wal_log_hints: "on" unix_socket_directories: '.' tags: - nofailover: False - noloadbalance: False - clonefrom: False - replicatefrom: postgresql1 + nofailover: false + noloadbalance: false + clonefrom: false + replicatefrom: postgres1 From b58ddc559ec3c2fff57dfbcc7c46178405ff4c55 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Tue, 23 Aug 2016 17:29:21 +0200 Subject: [PATCH 2/5] Do not drop active replication slots. Master tried to delete all slots that did not correspond to the replica registered in Patroni. That produced an error for the slots that were active, potentially preventing drop and creation of other slots. Reported by Murat Kabilov. --- patroni/postgresql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patroni/postgresql.py b/patroni/postgresql.py index 7dff8b32..e04b8f45 100644 --- a/patroni/postgresql.py +++ b/patroni/postgresql.py @@ -884,7 +884,7 @@ $$""".format(name, ' '.join(options)), name, password, password) for slot in set(self._replication_slots) - set(slots): self.query("""SELECT pg_drop_replication_slot(%s) WHERE EXISTS(SELECT 1 FROM pg_replication_slots - WHERE slot_name = %s)""", slot, slot) + WHERE slot_name = %s AND NOT active)""", slot, slot) # create new slots for slot in set(slots) - set(self._replication_slots): From 96da6340a942ef0959265d2bac3812565237e770 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 24 Aug 2016 09:46:56 +0200 Subject: [PATCH 3/5] Calculate future restart time dynamically (#268) `do_POST_restart` was ramdomly showing not 100% coverage after 2016-08-20 due to hardcoded timestamps. --- tests/test_api.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 1b36d34f..3e90346a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -188,7 +188,8 @@ class TestRestApiHandler(unittest.TestCase): post = request + '\nContent-Length: ' - def make_request(request): + def make_request(request=None, **kwargs): + request = json.dumps(kwargs) if request is None else request return '{0}{1}\n\n{2}'.format(post, len(request), request) # empty request @@ -198,26 +199,26 @@ class TestRestApiHandler(unittest.TestCase): request = make_request('foobar=baz') MockRestApiServer(RestApiHandler, request) # wrong role - request = make_request('{"schedule": "2016-08-20 12:45TZ+1", "role": "unknown", "postgres_version": "9.5.3"}') + request = make_request(schedule=future_restart_time.isoformat(), role='unknown', postgres_version='9.5.3') MockRestApiServer(RestApiHandler, request) # wrong version - request = make_request('{"schedule": "2016-08-20 12:45TZ+1", "role": "master", "postgres_version": "9.5.3.1"}') + request = make_request(schedule=future_restart_time.isoformat(), role='master', postgres_version='9.5.3.1') MockRestApiServer(RestApiHandler, request) # unknown filter - request = make_request('{"schedule": "2016-08-29 12:45TZ+1", "batman": "lives"}') + request = make_request(schedule=future_restart_time.isoformat(), batman='lives') MockRestApiServer(RestApiHandler, request) # incorrect schedule - request = make_request('{"schedule": "2016-08-42 12:45TZ+1", "role": "master"}') + request = make_request(schedule='2016-08-42 12:45TZ+1', role='master') MockRestApiServer(RestApiHandler, request) # everything fine, but the schedule is missing - request = make_request('{"role": "master", "postgres_version": "9.5.2"}') + request = make_request(role='master', postgres_version='9.5.2') MockRestApiServer(RestApiHandler, request) for retval in (True, False): with patch.object(MockHa, 'schedule_future_restart', Mock(return_value=retval)): - request = make_request('{"schedule": "2016-08-29 12:45TZ+1"}') + request = make_request(schedule=future_restart_time.isoformat()) MockRestApiServer(RestApiHandler, request) with patch.object(MockHa, 'restart', Mock(return_value=(retval, "foo"))): - request = make_request('{"role": "master", "postgres_version": "9.5.2"}') + request = make_request(role='master', postgres_version='9.5.2') MockRestApiServer(RestApiHandler, request) def test_do_DELETE_restart(self): From b472a3258952faa45e5d231e7f1abe15fbca5769 Mon Sep 17 00:00:00 2001 From: Murat Kabilov Date: Wed, 24 Aug 2016 12:46:25 +0200 Subject: [PATCH 4/5] Suppress error output on master check (#256) --- patroni/ha.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/patroni/ha.py b/patroni/ha.py index 3367572f..93567eed 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -172,8 +172,8 @@ class Ha(object): is_master = json['role'] == 'master' xlog_location = None if is_master else json['xlog']['replayed_location'] return (member, True, not is_master, xlog_location, json.get('tags', {})) - except: - logger.exception('request failed: GET %s', member.api_url) + except Exception as e: + logger.warning("request failed: GET %s (%s)", member.api_url, e) return (member, False, None, 0, {}) def fetch_nodes_statuses(self, members): From 05bcbe90eb5feab2e01c2a00a33abdeeb6884b4f Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 25 Aug 2016 10:43:50 +0200 Subject: [PATCH 5/5] close connection on start instead of stop (#272) --- patroni/postgresql.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/patroni/postgresql.py b/patroni/postgresql.py index 31f45148..da4b932a 100644 --- a/patroni/postgresql.py +++ b/patroni/postgresql.py @@ -503,6 +503,12 @@ class Postgresql(object): self._state = value def start(self, block_callbacks=False): + # make sure we close all connections established against + # the former node, otherwise, we might get a stalled one + # after kill -9, which would report incorrect data to + # patroni. + self.close_connection() + if self.is_running(): logger.error('Cannot start PostgreSQL because one is already running.') return True @@ -559,12 +565,6 @@ class Postgresql(object): return 'not accessible or not healty' def stop(self, mode='fast', block_callbacks=False, checkpoint=True): - # make sure we close all connections established against - # the former node, otherwise, we might get a stalled one - # after kill -9, which would report incorrect data to - # patroni. - - self.close_connection() if not self.is_running(): if not block_callbacks: self.set_state('stopped')