From f8bf1bb0abfff94c907fb8165be864add73b1e95 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 11 Apr 2016 16:03:41 +0200 Subject: [PATCH] Disable sudo, reshuffle travis tasks and introduce caching Without sudo travis is executing build tasks using docker and waiting time in this case is really small, usually not longer then 10 seconds. postgresql-9.5 is installed via addons.apt.packages (without sudo) But ports 5432 and 5433 are busy. So I had to ajust environment.py to assign port from higher diapason. And a few words about build tasks: First task is used for executing unit tests for all different python versions The second one is used for executing acceptance tests against etcd The third one is used for executing acceptance tests against zookeeper acceptance tests are executed with python2.7 and python3.5 In addition that I've introduced caching of python virtual environment. It really helps to reduce time needed to install python modules. --- .travis.yml | 82 ++++++++++++++++++++++++++--------------- features/environment.py | 13 ++++--- 2 files changed, 59 insertions(+), 36 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1fcd871e..93c7abcb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,43 +1,65 @@ -sudo: required +sudo: false language: python +python: +- "3.5" addons: + apt: + packages: + - postgresql-contrib-9.5 postgresql: "9.5" env: global: - - BOTO_CONFIG='' ETCDVERSION=2.2.5 ZKVERSION=3.4.6 + - ETCDVERSION=2.3.1 ZKVERSION=3.4.6 matrix: - - TEST_SUITE="python setup.py test" - - TEST_SUITE="behave" -python: - - "2.7" - - "3.4" - - "3.5" + - TEST_SUITE="python setup.py" + - DCS="etcd" TEST_SUITE="behave" + - DCS="exhibitor" TEST_SUITE="behave" +cache: + directories: + - $HOME/virtualenv/python2.7.9 + - $HOME/virtualenv/python3.4.2 + - $HOME/virtualenv/python3.5.0 install: - | + set -e + if [[ $TEST_SUITE == "behave" ]]; then - sudo bash -c ' - /etc/init.d/postgresql stop - apt-get -y remove --purge postgresql-9.1 postgresql-9.2 postgresql-9.3 postgresql-9.4 - apt-get -y autoremove - apt-key adv --keyserver keys.gnupg.net --recv-keys 7FCC7D46ACCC4CF8 - echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main 9.5" >> /etc/apt/sources.list.d/postgresql.list - apt-get update - apt-get -y install postgresql-9.5 - /etc/init.d/postgresql stop' - curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd - curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz - mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg - zookeeper-${ZKVERSION}/bin/zkServer.sh start - while true; do - echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \ - | nc -l 8181 &> /dev/null - done& + if [[ $DCS == "etcd" ]]; then + curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd + fi + + if [[ $DCS == "exhibitor" ]]; then + curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz + mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg + zookeeper-${ZKVERSION}/bin/zkServer.sh start + while true; do + echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' | nc -l 8181 &> /dev/null + done& + fi fi - - pip install -r requirements.txt - - pip install behave codacy-coverage coverage coveralls + + for pv in "2.7" "3.4" "3.5"; do + source ~/virtualenv/python${pv}/bin/activate + for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8 mock pytest-cov pytest'; do + pip install $p + done + done script: - - PATH=.:$PATH $TEST_SUITE - - if [[ $TEST_SUITE == "behave" ]]; then PATH=.:$PATH DCS=exhibitor $TEST_SUITE; else python setup.py flake8; fi + - | + for pv in "2.7" "3.4" "3.5"; do + source ~/virtualenv/python${pv}/bin/activate + + if [[ $TEST_SUITE == "behave" ]]; then + if [[ $pv != "3.4" ]]; then + if ! PATH=.:$PATH $TEST_SUITE; then + grep . features/output/*/*postgres?.* + fi + fi + else + $TEST_SUITE test + $TEST_SUITE flake8 + fi + done after_success: - coveralls - - if [[ -f coverage.xml ]]; then python-codacy-coverage -r coverage.xml; fi + - if [[ $TEST_SUITE != "behave" ]]; then python-codacy-coverage -r coverage.xml; fi diff --git a/features/environment.py b/features/environment.py index 52405eb6..851b458c 100644 --- a/features/environment.py +++ b/features/environment.py @@ -72,11 +72,13 @@ class AbstractController(object): class PatroniController(AbstractController): + __PORT = 5440 PATRONI_CONFIG = '{}.yml' """ starts and stops individual patronis""" def __init__(self, dcs, name, work_directory, output_dir, tags=None): super(PatroniController, self).__init__('patroni_' + name, work_directory, output_dir) + PatroniController.__PORT += 1 self._data_dir = os.path.join(work_directory, 'data', name) self._connstring = None self._config = self._make_patroni_test_config(name, dcs, tags) @@ -109,7 +111,11 @@ class PatroniController(AbstractController): with open(patroni_config_name) as f: config = yaml.load(f) - self._connstring = self._make_connstring(config) + host = config['postgresql']['listen'].split(':')[0] + + config['postgresql']['listen'] = config['postgresql']['connect_address'] = '{0}:{1}'.format(host, self.__PORT) + + self._connstring = 'host={0} port={1} dbname=postgres user=postgres'.format(host, self.__PORT) config['postgresql'].update({'name': name, 'data_dir': self._data_dir}) config['postgresql']['parameters'].update({ @@ -133,11 +139,6 @@ class PatroniController(AbstractController): return patroni_config_path - @staticmethod - def _make_connstring(config): - tmp = (config['postgresql']['connect_address'] + ':5432').split(':') - return 'host={0} port={1} dbname=postgres user=postgres'.format(*tmp[:2]) - def _connection(self): if not self._conn or self._conn.closed != 0: self._conn = psycopg2.connect(self._connstring)