Merge pull request #152 from zalando/feature/acceptance_tests_travis

Run acceptance tests on travis

.travis.yml

@@ -1,15 +1,29 @@
+sudo: required
 language: python
+addons:
+  postgresql: "9.5"
+env:
+  - BOTO_CONFIG=''
 python:
   - "2.7"
   - "3.4"
   - "3.5"
 install:
+  - sudo /etc/init.d/postgresql stop
+  - sudo apt-get -y remove --purge postgresql-9.1 postgresql-9.2 postgresql-9.3 postgresql-9.4
+  - sudo apt-get -y autoremove
+  - sudo apt-key adv --keyserver keys.gnupg.net --recv-keys 7FCC7D46ACCC4CF8
+  - sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main 9.5" >> /etc/apt/sources.list.d/postgresql.list'
+  - sudo apt-get update
+  - sudo apt-get -y install postgresql-9.5
+  - sudo /etc/init.d/postgresql stop
   - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt --use-mirrors; fi
   - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt; fi
   - pip install coveralls codacy-coverage
 script:
   - python setup.py test
   - python setup.py flake8
+  - bash -x acceptance_tests.sh
 after_success:
   - coveralls
   - python-codacy-coverage -r coverage.xml

acceptance_tests.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+ETCDVERSION=2.2.5
+
+BINDIR=bin
+[ -d $BINDIR ] || mkdir $BINDIR
+
+export PATH=$BINDIR:$PATH
+
+# Add etcd
+curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C $BINDIR --strip=1 --wildcards --no-anchored etcd etcdctl
+
+sudo pip2.7 install lettuce python-Levenshtein
+sudo pip2.7 install -r requirements-py2.txt
+
+exec lettuce

features/basic_replication.feature

@@ -11,6 +11,6 @@ Feature: basic replication
     When I kill postgres0
     Then postgres1 role is the primary after 30 seconds
     When I start postgres0
-    Then postgres0 role is the secondary after 10 seconds
+    Then postgres0 role is the secondary after 15 seconds
     When I add the table bar to postgres1
     Then table bar is present on postgres0 after 10 seconds

features/basic_replication.py

@@ -49,7 +49,8 @@ class BasicReplicationSteps(object):
     def check_role(step, pg_name, pg_role, max_promotion_timeout):
         '''(\w+) role is the (\w+) after (\d+) seconds'''
         if not world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)):
-            assert False, "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
+            assert False,\
+                "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
 
 
 BasicReplicationSteps(world)

features/cascading_replication.feature

@@ -8,5 +8,5 @@ Scenario: check a base backup from the replica
 	And I create label with "postgres0" in postgres0 data directory
 	And I create label with "postgres1" in postgres1 data directory
 	And I configure and start postgres2 with a tag clonefrom postgres1
-	Then replication works from postgres0 to postgres2 after 15 seconds
-	And there is a label with "postgres1" in postgres2 data directory
+	Then replication works from postgres0 to postgres2 after 30 seconds
+	And there is a label with "postgres0" in postgres2 data directory

features/patroni_api.feature

@@ -17,7 +17,7 @@ Scenario: check API requests on a stand-alone server
 	Then I receive a response code 500
 	And I receive a response text "failover is not possible: cluster does not have members except leader"
 	When I issue an empty POST request to http://127.0.0.1:8008/failover
-	Then I receive a response code 500
+	Then I receive a response code 400
 	And I receive a response text "No values given for required parameters leader and member"
 
 Scenario: check API requests for the primary-replica pair

features/patroni_api.py

@@ -75,7 +75,7 @@ class PatroniAPISteps(object):
         '''I receive a response (\w+) (.*)'''
         if component == 'code':
             assert self.status_code == int(data),\
-                    "status code {0} != {1}, response: {2}".format(self.status_code, int(data), self.response)
+                "status code {0} != {1}, response: {2}".format(self.status_code, int(data), self.response)
         elif component == 'text':
             assert self.response == data.strip('"'), "response {0} does not contain {1}".format(self.response, data)
         else:

features/terrain.py

@@ -120,7 +120,8 @@ class PatroniController(object):
             self.stop(patroni)
 
     def create_and_set_output_directory(self, feature_name):
-        feature_dir = os.path.join(pctl.patroni_path, "features", "output", feature_name.encode('utf-8').replace(' ', '_'))
+        feature_dir = os.path.join(pctl.patroni_path, "features", "output",
+                                   feature_name.encode('utf-8').replace(' ', '_'))
         if os.path.exists(feature_dir):
             shutil.rmtree(feature_dir)
         os.makedirs(feature_dir)
@@ -145,6 +146,7 @@ class PatroniController(object):
         postgresql_params['log_filename'] = '{0}.log'.format(pg_name)
         postgresql_params['log_statement'] = 'all'
         postgresql_params['log_min_messages'] = 'debug1'
+        postgresql_params['unix_socket_directories'] = '.'
 
         if tags:
             config['tags'] = {}
@@ -209,7 +211,10 @@ class EtcdController(object):
             return True
         self.work_directory = tempfile.mkdtemp()
         # etcd is running throughout the tests, no need to append to the log
-        self.log_file = open(os.path.join(self.log_directory, "features", "output", 'etcd.log'), 'w')
+        output_dir = os.path.join(self.log_directory, "features", "output")
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+        self.log_file = open(os.path.join(output_dir, 'etcd.log'), 'w')
         self.handle =\
             subprocess.Popen(["etcd", "--debug", "--data-dir", self.work_directory],
                              stdout=self.log_file, stderr=subprocess.STDOUT)

patroni/ha.py

@@ -457,8 +457,8 @@ class Ha(object):
             else:
                 # check if we are allowed to join
                 if self.sysid_valid(self.cluster.initialize) and self.cluster.initialize != self.state_handler.sysid:
-                    logger.fatal("system ID mismatch, node {0} belongs to a different cluster".
-                                 format(self.state_handler.name))
+                    logger.fatal("system ID mismatch, node %s belongs to a different cluster: %s != %s",
+                                 self.state_handler.name, self.cluster.initialize, self.state_handler.sysid)
                     sys.exit(1)
 
             # try to start dead postgres

patroni/postgresql.py

@@ -207,7 +207,7 @@ class Postgresql(object):
                 options.append('--username={0}'.format(self.superuser['username']))
             if 'password' in self.superuser:
                 (fd, pwfile) = tempfile.mkstemp()
-                os.write(fd, self.superuser['password'].encode())
+                os.write(fd, self.superuser['password'].encode('utf-8'))
                 os.close(fd)
                 options.append('--pwfile={0}'.format(pwfile))
 
@@ -506,7 +506,7 @@ recovery_target_timeline = 'latest'
         try:
             data = subprocess.check_output(['pg_controldata', self.data_dir])
             if data:
-                data = data.decode().splitlines()
+                data = data.decode('utf-8').splitlines()
                 result = {l.split(':')[0].replace('Current ', '', 1): l.split(':')[1].strip() for l in data if l}
         except subprocess.CalledProcessError:
             logger.exception("Error when calling pg_controldata")