Merge pull request #173 from zalando/feature/travis-build-matrix

Disable sudo, reshuffle travis tasks and introduce caching

.travis.yml

@@ -1,43 +1,75 @@
-sudo: required
+sudo: false
 language: python
+python:
+- "3.5"
 addons:
+  apt:
+    packages:
+    - postgresql-contrib-9.5
   postgresql: "9.5"
 env:
   global:
-    - BOTO_CONFIG='' ETCDVERSION=2.2.5 ZKVERSION=3.4.6
+  - ETCDVERSION=2.3.1 ZKVERSION=3.4.6
   matrix:
-    - TEST_SUITE="python setup.py test"
-    - TEST_SUITE="behave"
-python:
-  - "2.7"
-  - "3.4"
-  - "3.5"
+  - TEST_SUITE="python setup.py"
+  - DCS="etcd" TEST_SUITE="behave"
+  - DCS="exhibitor" TEST_SUITE="behave"
+cache:
+  directories:
+  - $HOME/virtualenv/python2.7.9
+  - $HOME/virtualenv/python3.4.2
+  - $HOME/virtualenv/python3.5.0
 install:
   - |
+    set -e
+
     if [[ $TEST_SUITE == "behave" ]]; then
-        sudo bash -c '
-            /etc/init.d/postgresql stop
-            apt-get -y remove --purge postgresql-9.1 postgresql-9.2 postgresql-9.3 postgresql-9.4
-            apt-get -y autoremove
-            apt-key adv --keyserver keys.gnupg.net --recv-keys 7FCC7D46ACCC4CF8
-            echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main 9.5" >> /etc/apt/sources.list.d/postgresql.list
-            apt-get update
-            apt-get -y install postgresql-9.5
-            /etc/init.d/postgresql stop'
-        curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd
-        curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz
-        mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg
-        zookeeper-${ZKVERSION}/bin/zkServer.sh start
-        while true; do
-            echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \
-              | nc -l 8181 &> /dev/null
-        done&
+        if [[ $DCS == "etcd" ]]; then
+            curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
+                | tar xz -C . --strip=1 --wildcards --no-anchored etcd
+        fi
+
+        if [[ $DCS == "exhibitor" ]]; then
+            curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz
+            mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg
+            zookeeper-${ZKVERSION}/bin/zkServer.sh start
+            # following lines are 'emulating' exhibitor REST API
+            while true; do
+                echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \
+                    | nc -l 8181 &> /dev/null
+            done&
+        fi
     fi
-  - pip install -r requirements.txt
-  - pip install behave codacy-coverage coverage coveralls
+
+    for pv in "2.7" "3.4" "3.5"; do
+        source ~/virtualenv/python${pv}/bin/activate
+        # explicitly install all needed python modules to cache them
+        for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8 mock pytest-cov pytest'; do
+            pip install $p
+        done
+    done
 script:
-  - PATH=.:$PATH $TEST_SUITE
-  - if [[ $TEST_SUITE == "behave" ]]; then PATH=.:$PATH DCS=exhibitor $TEST_SUITE; else python setup.py flake8; fi
+  - |
+    for pv in "2.7" "3.4" "3.5"; do
+        source ~/virtualenv/python${pv}/bin/activate
+
+        if [[ $TEST_SUITE == "behave" ]]; then
+            if [[ $pv != "3.4" ]]; then
+                echo Running acceptance tests using python${pv}
+                if ! PATH=.:$PATH $TEST_SUITE; then
+                    # output all log files when tests are failing
+                    grep . features/output/*/*postgres?.*
+                    exit 1
+                fi
+            fi
+        else
+            echo Running unit tests using python${pv}
+            $TEST_SUITE test
+            $TEST_SUITE flake8
+        fi
+    done
+
+    set +e
 after_success:
   - coveralls
-  - if [[ -f coverage.xml ]]; then python-codacy-coverage -r coverage.xml; fi
+  - if [[ $TEST_SUITE != "behave" ]]; then python-codacy-coverage -r coverage.xml; fi

features/basic_replication.feature

@@ -6,12 +6,12 @@ Feature: basic replication
     And postgres0 is a leader after 10 seconds
     And I start postgres1
     When I add the table foo to postgres0
-    Then table foo is present on postgres1 after 15 seconds
+    Then table foo is present on postgres1 after 20 seconds
 
   Scenario: check the basic failover
     When I kill postgres0
-    Then postgres1 role is the primary after 30 seconds
+    Then postgres1 role is the primary after 32 seconds
     When I start postgres0
-    Then postgres0 role is the secondary after 15 seconds
+    Then postgres0 role is the secondary after 20 seconds
     When I add the table bar to postgres1
-    Then table bar is present on postgres0 after 15 seconds
+    Then table bar is present on postgres0 after 20 seconds

features/cascading_replication.feature

@@ -5,7 +5,7 @@ Scenario: check a base backup and streaming replication from a replica
 	Given I start postgres0
 	And postgres0 is a leader after 10 seconds
 	And I configure and start postgres1 with a tag clonefrom true
-	And replication works from postgres0 to postgres1 after 15 seconds
+	And replication works from postgres0 to postgres1 after 20 seconds
 	And I create label with "postgres0" in postgres0 data directory
 	And I create label with "postgres1" in postgres1 data directory
 	And I configure and start postgres2 with a tag replicatefrom postgres1

features/environment.py

@@ -72,11 +72,13 @@ class AbstractController(object):
 
 
 class PatroniController(AbstractController):
+    __PORT = 5440
     PATRONI_CONFIG = '{}.yml'
     """ starts and stops individual patronis"""
 
     def __init__(self, dcs, name, work_directory, output_dir, tags=None):
         super(PatroniController, self).__init__('patroni_' + name, work_directory, output_dir)
+        PatroniController.__PORT += 1
         self._data_dir = os.path.join(work_directory, 'data', name)
         self._connstring = None
         self._config = self._make_patroni_test_config(name, dcs, tags)
@@ -109,7 +111,11 @@ class PatroniController(AbstractController):
         with open(patroni_config_name) as f:
             config = yaml.load(f)
 
-        self._connstring = self._make_connstring(config)
+        host = config['postgresql']['listen'].split(':')[0]
+
+        config['postgresql']['listen'] = config['postgresql']['connect_address'] = '{0}:{1}'.format(host, self.__PORT)
+
+        self._connstring = 'host={0} port={1} dbname=postgres user=postgres'.format(host, self.__PORT)
 
         config['postgresql'].update({'name': name, 'data_dir': self._data_dir})
         config['postgresql']['parameters'].update({
@@ -133,11 +139,6 @@ class PatroniController(AbstractController):
 
         return patroni_config_path
 
-    @staticmethod
-    def _make_connstring(config):
-        tmp = (config['postgresql']['connect_address'] + ':5432').split(':')
-        return 'host={0} port={1} dbname=postgres user=postgres'.format(*tmp[:2])
-
     def _connection(self):
         if not self._conn or self._conn.closed != 0:
             self._conn = psycopg2.connect(self._connstring)

features/patroni_api.feature

@@ -22,7 +22,7 @@ Scenario: check API requests on a stand-alone server
 
 Scenario: check API requests for the primary-replica pair
 	Given I start postgres1
-	And replication works from postgres0 to postgres1 after 15 seconds
+	And replication works from postgres0 to postgres1 after 20 seconds
 	When I issue a GET request to http://127.0.0.1:8009/replica
 	Then I receive a response code 200
 	And I receive a response state running
@@ -40,12 +40,14 @@ Scenario: check the failover via the API
 	Then I receive a response code 200
 	And postgres1 is a leader after 5 seconds
         And postgres1 role is the primary after 5 seconds
-        And postgres0 role is the secondary after 5 seconds
-	And replication works from postgres1 to postgres0 after 15 seconds
+        And postgres0 role is the secondary after 10 seconds
+	And replication works from postgres1 to postgres0 after 20 seconds
 
 Scenario: check the scheduled failover
-	Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgres0 in 10 seconds
+	Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgres0 in 1 seconds
 	Then I receive a response code 200
 	And postgres0 is a leader after 20 seconds
+        And postgres0 role is the primary after 5 seconds
+        And postgres1 role is the secondary after 10 seconds
 	And replication works from postgres0 to postgres1 after 25 seconds
 

patroni/api.py

@@ -203,8 +203,12 @@ class RestApiHandler(BaseHTTPRequestHandler):
                         data = b'Cannot schedule failover in the past'
                         status_code = 422
                     elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
+                        self.server.patroni.dcs.event.set()
                         data = b'Failover scheduled'
                         status_code = 200
+                    else:
+                        data = b'failed to write failover key into DCS'
+                        status_code = 503
                 except (ValueError, TypeError):
                     logger.exception('Invalid scheduled failover time: %s', request['scheduled_at'])
                     data = b'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601'
@@ -212,12 +216,12 @@ class RestApiHandler(BaseHTTPRequestHandler):
             else:
                 data = self.is_failover_possible(cluster, leader, candidate)
                 if not data:
-                    if not self.server.patroni.dcs.manual_failover(leader, candidate):
-                        data = b'failed to write failover key into DCS'
-                        status_code = 503
-                    else:
+                    if self.server.patroni.dcs.manual_failover(leader, candidate):
                         self.server.patroni.dcs.event.set()
                         status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, candidate)
+                    else:
+                        data = b'failed to write failover key into DCS'
+                        status_code = 503
         else:
             status_code = 400
             data = b'No values given for required parameters leader and candidate'

patroni/ha.py

@@ -38,7 +38,7 @@ class Ha(object):
 
     def update_lock(self):
         ret = self.dcs.update_leader()
-        if ret:
+        if ret and not self._async_executor.busy:
             try:
                 self.dcs.write_leader_optime(self.state_handler.last_operation())
             except:
@@ -59,7 +59,7 @@ class Ha(object):
         }
         if self.patroni.tags:
             data['tags'] = self.patroni.tags
-        if data['state'] in ['running', 'restarting', 'starting']:
+        if not self._async_executor.busy and data['state'] in ['running', 'restarting', 'starting']:
             try:
                 data['xlog_location'] = self.state_handler.xlog_position()
             except:

setup.py

@@ -93,7 +93,12 @@ class PyTest(TestCommand):
             params['args'] += self.cov
         if self.junitxml:
             params['args'] += self.junitxml
-        params['args'] += ['--doctest-modules', MAIN_PACKAGE, '-s', '-vv']
+        params['args'] += ['--doctest-modules', MAIN_PACKAGE, '-vv']
+
+        import logging
+        silence = logging.WARNING
+        logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=os.getenv('LOGLEVEL', silence))
+        params['args'] += ['-s' if logging.getLogger().getEffectiveLevel() < silence else '--capture=fd']
         errno = pytest.main(**params)
         sys.exit(errno)
 

tests/test_api.py

@@ -188,6 +188,9 @@ class TestRestApiHandler(unittest.TestCase):
         request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\
                   b'"postgresql1", "member": "postgresql2", "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}'
         MockRestApiServer(RestApiHandler, request)
+        with patch.object(MockPatroni, 'dcs') as d:
+            d.manual_failover.return_value = False
+            MockRestApiServer(RestApiHandler, request)
 
         # Exception: No timezone specified
         request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 97\n\n{"leader": ' +\