diff --git a/.travis.yml b/.travis.yml index 1fcd871e..c4324674 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,43 +1,75 @@ -sudo: required +sudo: false language: python +python: +- "3.5" addons: + apt: + packages: + - postgresql-contrib-9.5 postgresql: "9.5" env: global: - - BOTO_CONFIG='' ETCDVERSION=2.2.5 ZKVERSION=3.4.6 + - ETCDVERSION=2.3.1 ZKVERSION=3.4.6 matrix: - - TEST_SUITE="python setup.py test" - - TEST_SUITE="behave" -python: - - "2.7" - - "3.4" - - "3.5" + - TEST_SUITE="python setup.py" + - DCS="etcd" TEST_SUITE="behave" + - DCS="exhibitor" TEST_SUITE="behave" +cache: + directories: + - $HOME/virtualenv/python2.7.9 + - $HOME/virtualenv/python3.4.2 + - $HOME/virtualenv/python3.5.0 install: - | + set -e + if [[ $TEST_SUITE == "behave" ]]; then - sudo bash -c ' - /etc/init.d/postgresql stop - apt-get -y remove --purge postgresql-9.1 postgresql-9.2 postgresql-9.3 postgresql-9.4 - apt-get -y autoremove - apt-key adv --keyserver keys.gnupg.net --recv-keys 7FCC7D46ACCC4CF8 - echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main 9.5" >> /etc/apt/sources.list.d/postgresql.list - apt-get update - apt-get -y install postgresql-9.5 - /etc/init.d/postgresql stop' - curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd - curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz - mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg - zookeeper-${ZKVERSION}/bin/zkServer.sh start - while true; do - echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \ - | nc -l 8181 &> /dev/null - done& + if [[ $DCS == "etcd" ]]; then + curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \ + | tar xz -C . --strip=1 --wildcards --no-anchored etcd + fi + + if [[ $DCS == "exhibitor" ]]; then + curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz + mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg + zookeeper-${ZKVERSION}/bin/zkServer.sh start + # following lines are 'emulating' exhibitor REST API + while true; do + echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \ + | nc -l 8181 &> /dev/null + done& + fi fi - - pip install -r requirements.txt - - pip install behave codacy-coverage coverage coveralls + + for pv in "2.7" "3.4" "3.5"; do + source ~/virtualenv/python${pv}/bin/activate + # explicitly install all needed python modules to cache them + for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8 mock pytest-cov pytest'; do + pip install $p + done + done script: - - PATH=.:$PATH $TEST_SUITE - - if [[ $TEST_SUITE == "behave" ]]; then PATH=.:$PATH DCS=exhibitor $TEST_SUITE; else python setup.py flake8; fi + - | + for pv in "2.7" "3.4" "3.5"; do + source ~/virtualenv/python${pv}/bin/activate + + if [[ $TEST_SUITE == "behave" ]]; then + if [[ $pv != "3.4" ]]; then + echo Running acceptance tests using python${pv} + if ! PATH=.:$PATH $TEST_SUITE; then + # output all log files when tests are failing + grep . features/output/*/*postgres?.* + exit 1 + fi + fi + else + echo Running unit tests using python${pv} + $TEST_SUITE test + $TEST_SUITE flake8 + fi + done + + set +e after_success: - coveralls - - if [[ -f coverage.xml ]]; then python-codacy-coverage -r coverage.xml; fi + - if [[ $TEST_SUITE != "behave" ]]; then python-codacy-coverage -r coverage.xml; fi diff --git a/features/basic_replication.feature b/features/basic_replication.feature index 553440c4..be192715 100644 --- a/features/basic_replication.feature +++ b/features/basic_replication.feature @@ -6,12 +6,12 @@ Feature: basic replication And postgres0 is a leader after 10 seconds And I start postgres1 When I add the table foo to postgres0 - Then table foo is present on postgres1 after 15 seconds + Then table foo is present on postgres1 after 20 seconds Scenario: check the basic failover When I kill postgres0 - Then postgres1 role is the primary after 30 seconds + Then postgres1 role is the primary after 32 seconds When I start postgres0 - Then postgres0 role is the secondary after 15 seconds + Then postgres0 role is the secondary after 20 seconds When I add the table bar to postgres1 - Then table bar is present on postgres0 after 15 seconds + Then table bar is present on postgres0 after 20 seconds diff --git a/features/cascading_replication.feature b/features/cascading_replication.feature index 27f72eba..4a1672df 100644 --- a/features/cascading_replication.feature +++ b/features/cascading_replication.feature @@ -5,7 +5,7 @@ Scenario: check a base backup and streaming replication from a replica Given I start postgres0 And postgres0 is a leader after 10 seconds And I configure and start postgres1 with a tag clonefrom true - And replication works from postgres0 to postgres1 after 15 seconds + And replication works from postgres0 to postgres1 after 20 seconds And I create label with "postgres0" in postgres0 data directory And I create label with "postgres1" in postgres1 data directory And I configure and start postgres2 with a tag replicatefrom postgres1 diff --git a/features/environment.py b/features/environment.py index 52405eb6..851b458c 100644 --- a/features/environment.py +++ b/features/environment.py @@ -72,11 +72,13 @@ class AbstractController(object): class PatroniController(AbstractController): + __PORT = 5440 PATRONI_CONFIG = '{}.yml' """ starts and stops individual patronis""" def __init__(self, dcs, name, work_directory, output_dir, tags=None): super(PatroniController, self).__init__('patroni_' + name, work_directory, output_dir) + PatroniController.__PORT += 1 self._data_dir = os.path.join(work_directory, 'data', name) self._connstring = None self._config = self._make_patroni_test_config(name, dcs, tags) @@ -109,7 +111,11 @@ class PatroniController(AbstractController): with open(patroni_config_name) as f: config = yaml.load(f) - self._connstring = self._make_connstring(config) + host = config['postgresql']['listen'].split(':')[0] + + config['postgresql']['listen'] = config['postgresql']['connect_address'] = '{0}:{1}'.format(host, self.__PORT) + + self._connstring = 'host={0} port={1} dbname=postgres user=postgres'.format(host, self.__PORT) config['postgresql'].update({'name': name, 'data_dir': self._data_dir}) config['postgresql']['parameters'].update({ @@ -133,11 +139,6 @@ class PatroniController(AbstractController): return patroni_config_path - @staticmethod - def _make_connstring(config): - tmp = (config['postgresql']['connect_address'] + ':5432').split(':') - return 'host={0} port={1} dbname=postgres user=postgres'.format(*tmp[:2]) - def _connection(self): if not self._conn or self._conn.closed != 0: self._conn = psycopg2.connect(self._connstring) diff --git a/features/patroni_api.feature b/features/patroni_api.feature index d3b3130d..31bbbdb2 100644 --- a/features/patroni_api.feature +++ b/features/patroni_api.feature @@ -22,7 +22,7 @@ Scenario: check API requests on a stand-alone server Scenario: check API requests for the primary-replica pair Given I start postgres1 - And replication works from postgres0 to postgres1 after 15 seconds + And replication works from postgres0 to postgres1 after 20 seconds When I issue a GET request to http://127.0.0.1:8009/replica Then I receive a response code 200 And I receive a response state running @@ -40,12 +40,14 @@ Scenario: check the failover via the API Then I receive a response code 200 And postgres1 is a leader after 5 seconds And postgres1 role is the primary after 5 seconds - And postgres0 role is the secondary after 5 seconds - And replication works from postgres1 to postgres0 after 15 seconds + And postgres0 role is the secondary after 10 seconds + And replication works from postgres1 to postgres0 after 20 seconds Scenario: check the scheduled failover - Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgres0 in 10 seconds + Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgres0 in 1 seconds Then I receive a response code 200 And postgres0 is a leader after 20 seconds + And postgres0 role is the primary after 5 seconds + And postgres1 role is the secondary after 10 seconds And replication works from postgres0 to postgres1 after 25 seconds diff --git a/patroni/api.py b/patroni/api.py index 2ae66e40..725faf0f 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -203,8 +203,12 @@ class RestApiHandler(BaseHTTPRequestHandler): data = b'Cannot schedule failover in the past' status_code = 422 elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at): + self.server.patroni.dcs.event.set() data = b'Failover scheduled' status_code = 200 + else: + data = b'failed to write failover key into DCS' + status_code = 503 except (ValueError, TypeError): logger.exception('Invalid scheduled failover time: %s', request['scheduled_at']) data = b'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601' @@ -212,12 +216,12 @@ class RestApiHandler(BaseHTTPRequestHandler): else: data = self.is_failover_possible(cluster, leader, candidate) if not data: - if not self.server.patroni.dcs.manual_failover(leader, candidate): - data = b'failed to write failover key into DCS' - status_code = 503 - else: + if self.server.patroni.dcs.manual_failover(leader, candidate): self.server.patroni.dcs.event.set() status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, candidate) + else: + data = b'failed to write failover key into DCS' + status_code = 503 else: status_code = 400 data = b'No values given for required parameters leader and candidate' diff --git a/patroni/ha.py b/patroni/ha.py index 21999304..6923a126 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -38,7 +38,7 @@ class Ha(object): def update_lock(self): ret = self.dcs.update_leader() - if ret: + if ret and not self._async_executor.busy: try: self.dcs.write_leader_optime(self.state_handler.last_operation()) except: @@ -59,7 +59,7 @@ class Ha(object): } if self.patroni.tags: data['tags'] = self.patroni.tags - if data['state'] in ['running', 'restarting', 'starting']: + if not self._async_executor.busy and data['state'] in ['running', 'restarting', 'starting']: try: data['xlog_location'] = self.state_handler.xlog_position() except: diff --git a/setup.py b/setup.py index 2050baff..6af7b953 100644 --- a/setup.py +++ b/setup.py @@ -93,7 +93,12 @@ class PyTest(TestCommand): params['args'] += self.cov if self.junitxml: params['args'] += self.junitxml - params['args'] += ['--doctest-modules', MAIN_PACKAGE, '-s', '-vv'] + params['args'] += ['--doctest-modules', MAIN_PACKAGE, '-vv'] + + import logging + silence = logging.WARNING + logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=os.getenv('LOGLEVEL', silence)) + params['args'] += ['-s' if logging.getLogger().getEffectiveLevel() < silence else '--capture=fd'] errno = pytest.main(**params) sys.exit(errno) diff --git a/tests/test_api.py b/tests/test_api.py index 1a3d00ce..5b31104e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -188,6 +188,9 @@ class TestRestApiHandler(unittest.TestCase): request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\ b'"postgresql1", "member": "postgresql2", "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}' MockRestApiServer(RestApiHandler, request) + with patch.object(MockPatroni, 'dcs') as d: + d.manual_failover.return_value = False + MockRestApiServer(RestApiHandler, request) # Exception: No timezone specified request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 97\n\n{"leader": ' +\