From f8bf1bb0abfff94c907fb8165be864add73b1e95 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 11 Apr 2016 16:03:41 +0200 Subject: [PATCH 01/11] Disable sudo, reshuffle travis tasks and introduce caching Without sudo travis is executing build tasks using docker and waiting time in this case is really small, usually not longer then 10 seconds. postgresql-9.5 is installed via addons.apt.packages (without sudo) But ports 5432 and 5433 are busy. So I had to ajust environment.py to assign port from higher diapason. And a few words about build tasks: First task is used for executing unit tests for all different python versions The second one is used for executing acceptance tests against etcd The third one is used for executing acceptance tests against zookeeper acceptance tests are executed with python2.7 and python3.5 In addition that I've introduced caching of python virtual environment. It really helps to reduce time needed to install python modules. --- .travis.yml | 82 ++++++++++++++++++++++++++--------------- features/environment.py | 13 ++++--- 2 files changed, 59 insertions(+), 36 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1fcd871e..93c7abcb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,43 +1,65 @@ -sudo: required +sudo: false language: python +python: +- "3.5" addons: + apt: + packages: + - postgresql-contrib-9.5 postgresql: "9.5" env: global: - - BOTO_CONFIG='' ETCDVERSION=2.2.5 ZKVERSION=3.4.6 + - ETCDVERSION=2.3.1 ZKVERSION=3.4.6 matrix: - - TEST_SUITE="python setup.py test" - - TEST_SUITE="behave" -python: - - "2.7" - - "3.4" - - "3.5" + - TEST_SUITE="python setup.py" + - DCS="etcd" TEST_SUITE="behave" + - DCS="exhibitor" TEST_SUITE="behave" +cache: + directories: + - $HOME/virtualenv/python2.7.9 + - $HOME/virtualenv/python3.4.2 + - $HOME/virtualenv/python3.5.0 install: - | + set -e + if [[ $TEST_SUITE == "behave" ]]; then - sudo bash -c ' - /etc/init.d/postgresql stop - apt-get -y remove --purge postgresql-9.1 postgresql-9.2 postgresql-9.3 postgresql-9.4 - apt-get -y autoremove - apt-key adv --keyserver keys.gnupg.net --recv-keys 7FCC7D46ACCC4CF8 - echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main 9.5" >> /etc/apt/sources.list.d/postgresql.list - apt-get update - apt-get -y install postgresql-9.5 - /etc/init.d/postgresql stop' - curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd - curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz - mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg - zookeeper-${ZKVERSION}/bin/zkServer.sh start - while true; do - echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \ - | nc -l 8181 &> /dev/null - done& + if [[ $DCS == "etcd" ]]; then + curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd + fi + + if [[ $DCS == "exhibitor" ]]; then + curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz + mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg + zookeeper-${ZKVERSION}/bin/zkServer.sh start + while true; do + echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' | nc -l 8181 &> /dev/null + done& + fi fi - - pip install -r requirements.txt - - pip install behave codacy-coverage coverage coveralls + + for pv in "2.7" "3.4" "3.5"; do + source ~/virtualenv/python${pv}/bin/activate + for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8 mock pytest-cov pytest'; do + pip install $p + done + done script: - - PATH=.:$PATH $TEST_SUITE - - if [[ $TEST_SUITE == "behave" ]]; then PATH=.:$PATH DCS=exhibitor $TEST_SUITE; else python setup.py flake8; fi + - | + for pv in "2.7" "3.4" "3.5"; do + source ~/virtualenv/python${pv}/bin/activate + + if [[ $TEST_SUITE == "behave" ]]; then + if [[ $pv != "3.4" ]]; then + if ! PATH=.:$PATH $TEST_SUITE; then + grep . features/output/*/*postgres?.* + fi + fi + else + $TEST_SUITE test + $TEST_SUITE flake8 + fi + done after_success: - coveralls - - if [[ -f coverage.xml ]]; then python-codacy-coverage -r coverage.xml; fi + - if [[ $TEST_SUITE != "behave" ]]; then python-codacy-coverage -r coverage.xml; fi diff --git a/features/environment.py b/features/environment.py index 52405eb6..851b458c 100644 --- a/features/environment.py +++ b/features/environment.py @@ -72,11 +72,13 @@ class AbstractController(object): class PatroniController(AbstractController): + __PORT = 5440 PATRONI_CONFIG = '{}.yml' """ starts and stops individual patronis""" def __init__(self, dcs, name, work_directory, output_dir, tags=None): super(PatroniController, self).__init__('patroni_' + name, work_directory, output_dir) + PatroniController.__PORT += 1 self._data_dir = os.path.join(work_directory, 'data', name) self._connstring = None self._config = self._make_patroni_test_config(name, dcs, tags) @@ -109,7 +111,11 @@ class PatroniController(AbstractController): with open(patroni_config_name) as f: config = yaml.load(f) - self._connstring = self._make_connstring(config) + host = config['postgresql']['listen'].split(':')[0] + + config['postgresql']['listen'] = config['postgresql']['connect_address'] = '{0}:{1}'.format(host, self.__PORT) + + self._connstring = 'host={0} port={1} dbname=postgres user=postgres'.format(host, self.__PORT) config['postgresql'].update({'name': name, 'data_dir': self._data_dir}) config['postgresql']['parameters'].update({ @@ -133,11 +139,6 @@ class PatroniController(AbstractController): return patroni_config_path - @staticmethod - def _make_connstring(config): - tmp = (config['postgresql']['connect_address'] + ':5432').split(':') - return 'host={0} port={1} dbname=postgres user=postgres'.format(*tmp[:2]) - def _connection(self): if not self._conn or self._conn.closed != 0: self._conn = psycopg2.connect(self._connstring) From 15d30a2d359de13bf253229bec72666125d38051 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 13 Apr 2016 08:32:24 +0200 Subject: [PATCH 02/11] Try to stabilize acceptance tests --- .travis.yml | 4 ++++ features/patroni_api.feature | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 93c7abcb..3e43240f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,15 +51,19 @@ script: if [[ $TEST_SUITE == "behave" ]]; then if [[ $pv != "3.4" ]]; then + echo Running acceptance tests using python${pv} if ! PATH=.:$PATH $TEST_SUITE; then grep . features/output/*/*postgres?.* fi fi else + echo Running unit tests using python${pv} $TEST_SUITE test $TEST_SUITE flake8 fi done + + set +e after_success: - coveralls - if [[ $TEST_SUITE != "behave" ]]; then python-codacy-coverage -r coverage.xml; fi diff --git a/features/patroni_api.feature b/features/patroni_api.feature index d3b3130d..75d44304 100644 --- a/features/patroni_api.feature +++ b/features/patroni_api.feature @@ -40,12 +40,14 @@ Scenario: check the failover via the API Then I receive a response code 200 And postgres1 is a leader after 5 seconds And postgres1 role is the primary after 5 seconds - And postgres0 role is the secondary after 5 seconds + And postgres0 role is the secondary after 10 seconds And replication works from postgres1 to postgres0 after 15 seconds Scenario: check the scheduled failover Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgres0 in 10 seconds Then I receive a response code 200 And postgres0 is a leader after 20 seconds + And postgres0 role is the primary after 5 seconds + And postgres1 role is the secondary after 10 seconds And replication works from postgres0 to postgres1 after 25 seconds From ff41818a84784445da8167e255ecb5221711382b Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 13 Apr 2016 09:24:43 +0200 Subject: [PATCH 03/11] Make unit-tests output less verbose It will capture output to stdout and stderr and print it when test is failed. Please set LOGLEVEL env variable to INFO or DEBUG if you want to see everything (as it was before). --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2050baff..6af7b953 100644 --- a/setup.py +++ b/setup.py @@ -93,7 +93,12 @@ class PyTest(TestCommand): params['args'] += self.cov if self.junitxml: params['args'] += self.junitxml - params['args'] += ['--doctest-modules', MAIN_PACKAGE, '-s', '-vv'] + params['args'] += ['--doctest-modules', MAIN_PACKAGE, '-vv'] + + import logging + silence = logging.WARNING + logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=os.getenv('LOGLEVEL', silence)) + params['args'] += ['-s' if logging.getLogger().getEffectiveLevel() < silence else '--capture=fd'] errno = pytest.main(**params) sys.exit(errno) From 94331fde6dbb94159bcb2f0585129745f4f2345d Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 13 Apr 2016 10:57:28 +0200 Subject: [PATCH 04/11] Do not try to fetch xlog_location when async_executor is busy --- patroni/ha.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patroni/ha.py b/patroni/ha.py index 21999304..7cdecd02 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -59,7 +59,7 @@ class Ha(object): } if self.patroni.tags: data['tags'] = self.patroni.tags - if data['state'] in ['running', 'restarting', 'starting']: + if not self._async_executor.busy and data['state'] in ['running', 'restarting', 'starting']: try: data['xlog_location'] = self.state_handler.xlog_position() except: From b4e86f080990368a931fefa56a62ec35ae3891d6 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 13 Apr 2016 11:00:32 +0200 Subject: [PATCH 05/11] Make it possible to schedule failover in less then 10 seconds But only when API request was posted to the leader --- features/patroni_api.feature | 2 +- patroni/api.py | 12 ++++++++---- tests/test_api.py | 3 +++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/features/patroni_api.feature b/features/patroni_api.feature index 75d44304..9216343e 100644 --- a/features/patroni_api.feature +++ b/features/patroni_api.feature @@ -44,7 +44,7 @@ Scenario: check the failover via the API And replication works from postgres1 to postgres0 after 15 seconds Scenario: check the scheduled failover - Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgres0 in 10 seconds + Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgres0 in 1 seconds Then I receive a response code 200 And postgres0 is a leader after 20 seconds And postgres0 role is the primary after 5 seconds diff --git a/patroni/api.py b/patroni/api.py index 2ae66e40..725faf0f 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -203,8 +203,12 @@ class RestApiHandler(BaseHTTPRequestHandler): data = b'Cannot schedule failover in the past' status_code = 422 elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at): + self.server.patroni.dcs.event.set() data = b'Failover scheduled' status_code = 200 + else: + data = b'failed to write failover key into DCS' + status_code = 503 except (ValueError, TypeError): logger.exception('Invalid scheduled failover time: %s', request['scheduled_at']) data = b'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601' @@ -212,12 +216,12 @@ class RestApiHandler(BaseHTTPRequestHandler): else: data = self.is_failover_possible(cluster, leader, candidate) if not data: - if not self.server.patroni.dcs.manual_failover(leader, candidate): - data = b'failed to write failover key into DCS' - status_code = 503 - else: + if self.server.patroni.dcs.manual_failover(leader, candidate): self.server.patroni.dcs.event.set() status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, candidate) + else: + data = b'failed to write failover key into DCS' + status_code = 503 else: status_code = 400 data = b'No values given for required parameters leader and candidate' diff --git a/tests/test_api.py b/tests/test_api.py index 1a3d00ce..5b31104e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -188,6 +188,9 @@ class TestRestApiHandler(unittest.TestCase): request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\ b'"postgresql1", "member": "postgresql2", "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}' MockRestApiServer(RestApiHandler, request) + with patch.object(MockPatroni, 'dcs') as d: + d.manual_failover.return_value = False + MockRestApiServer(RestApiHandler, request) # Exception: No timezone specified request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 97\n\n{"leader": ' +\ From 0ebfb30192eeb1e3d03dab7c4d00d0d165b705a5 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 13 Apr 2016 11:02:24 +0200 Subject: [PATCH 06/11] break on test failure --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3e43240f..dce48db2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,6 +54,7 @@ script: echo Running acceptance tests using python${pv} if ! PATH=.:$PATH $TEST_SUITE; then grep . features/output/*/*postgres?.* + exit 1 fi fi else From 01da5266a05f9c28cfe18b8d9886376dabfcfbd6 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 13 Apr 2016 11:11:14 +0200 Subject: [PATCH 07/11] Give time for running healh-checks when promoting replica --- features/basic_replication.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/basic_replication.feature b/features/basic_replication.feature index 553440c4..86b8c42f 100644 --- a/features/basic_replication.feature +++ b/features/basic_replication.feature @@ -10,7 +10,7 @@ Feature: basic replication Scenario: check the basic failover When I kill postgres0 - Then postgres1 role is the primary after 30 seconds + Then postgres1 role is the primary after 32 seconds When I start postgres0 Then postgres0 role is the secondary after 15 seconds When I add the table bar to postgres1 From 946d8ef4f241133feafd371cdf31e130eda7b100 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 13 Apr 2016 13:04:51 +0200 Subject: [PATCH 08/11] Do not try to query xlog_location when async_executor is busy --- patroni/ha.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patroni/ha.py b/patroni/ha.py index 7cdecd02..6923a126 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -38,7 +38,7 @@ class Ha(object): def update_lock(self): ret = self.dcs.update_leader() - if ret: + if ret and not self._async_executor.busy: try: self.dcs.write_leader_optime(self.state_handler.last_operation()) except: From d57310bbc02088aa22b935cd430b49a917b68c19 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 13 Apr 2016 14:09:45 +0200 Subject: [PATCH 09/11] Fix one more corner-case It could take up to 10 seconds to create replication slot. In addition to that when replica fails to connect to the master via streaming replication it doesn't retry immediately, but with some timeout (5 seconds). 10 + 5 == 15 what causes replication check scenarios fail. --- features/basic_replication.feature | 4 ++-- features/cascading_replication.feature | 2 +- features/patroni_api.feature | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/features/basic_replication.feature b/features/basic_replication.feature index 86b8c42f..d2995e14 100644 --- a/features/basic_replication.feature +++ b/features/basic_replication.feature @@ -6,7 +6,7 @@ Feature: basic replication And postgres0 is a leader after 10 seconds And I start postgres1 When I add the table foo to postgres0 - Then table foo is present on postgres1 after 15 seconds + Then table foo is present on postgres1 after 20 seconds Scenario: check the basic failover When I kill postgres0 @@ -14,4 +14,4 @@ Feature: basic replication When I start postgres0 Then postgres0 role is the secondary after 15 seconds When I add the table bar to postgres1 - Then table bar is present on postgres0 after 15 seconds + Then table bar is present on postgres0 after 20 seconds diff --git a/features/cascading_replication.feature b/features/cascading_replication.feature index 27f72eba..4a1672df 100644 --- a/features/cascading_replication.feature +++ b/features/cascading_replication.feature @@ -5,7 +5,7 @@ Scenario: check a base backup and streaming replication from a replica Given I start postgres0 And postgres0 is a leader after 10 seconds And I configure and start postgres1 with a tag clonefrom true - And replication works from postgres0 to postgres1 after 15 seconds + And replication works from postgres0 to postgres1 after 20 seconds And I create label with "postgres0" in postgres0 data directory And I create label with "postgres1" in postgres1 data directory And I configure and start postgres2 with a tag replicatefrom postgres1 diff --git a/features/patroni_api.feature b/features/patroni_api.feature index 9216343e..31bbbdb2 100644 --- a/features/patroni_api.feature +++ b/features/patroni_api.feature @@ -22,7 +22,7 @@ Scenario: check API requests on a stand-alone server Scenario: check API requests for the primary-replica pair Given I start postgres1 - And replication works from postgres0 to postgres1 after 15 seconds + And replication works from postgres0 to postgres1 after 20 seconds When I issue a GET request to http://127.0.0.1:8009/replica Then I receive a response code 200 And I receive a response state running @@ -41,7 +41,7 @@ Scenario: check the failover via the API And postgres1 is a leader after 5 seconds And postgres1 role is the primary after 5 seconds And postgres0 role is the secondary after 10 seconds - And replication works from postgres1 to postgres0 after 15 seconds + And replication works from postgres1 to postgres0 after 20 seconds Scenario: check the scheduled failover Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgres0 in 1 seconds From 7006a4ee147801a8ac30607ddc230bc0efdf60ea Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 13 Apr 2016 14:28:00 +0200 Subject: [PATCH 10/11] Sometimes replica can't attach to the master after pg_rewind The reason for that is: it takes up to 10 seconds to create replication slot + up to 5 seconds to start straming and recover. --- features/basic_replication.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/basic_replication.feature b/features/basic_replication.feature index d2995e14..be192715 100644 --- a/features/basic_replication.feature +++ b/features/basic_replication.feature @@ -12,6 +12,6 @@ Feature: basic replication When I kill postgres0 Then postgres1 role is the primary after 32 seconds When I start postgres0 - Then postgres0 role is the secondary after 15 seconds + Then postgres0 role is the secondary after 20 seconds When I add the table bar to postgres1 Then table bar is present on postgres0 after 20 seconds From 6dc4aacf05359924dfd33ebb9daac654adc0d328 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 14 Apr 2016 13:06:42 +0200 Subject: [PATCH 11/11] Add some comments to .travis.yml --- .travis.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index dce48db2..c4324674 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,21 +25,25 @@ install: if [[ $TEST_SUITE == "behave" ]]; then if [[ $DCS == "etcd" ]]; then - curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd + curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \ + | tar xz -C . --strip=1 --wildcards --no-anchored etcd fi if [[ $DCS == "exhibitor" ]]; then curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg zookeeper-${ZKVERSION}/bin/zkServer.sh start + # following lines are 'emulating' exhibitor REST API while true; do - echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' | nc -l 8181 &> /dev/null + echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \ + | nc -l 8181 &> /dev/null done& fi fi for pv in "2.7" "3.4" "3.5"; do source ~/virtualenv/python${pv}/bin/activate + # explicitly install all needed python modules to cache them for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8 mock pytest-cov pytest'; do pip install $p done @@ -53,6 +57,7 @@ script: if [[ $pv != "3.4" ]]; then echo Running acceptance tests using python${pv} if ! PATH=.:$PATH $TEST_SUITE; then + # output all log files when tests are failing grep . features/output/*/*postgres?.* exit 1 fi