Increase timeouts when running AT on travis (#324)

* Increase timeouts two times when running AT on travis
* Make up to 3 attempts to download DCS
* Get rid from hard-coded names

.travis.yml

@@ -9,7 +9,7 @@ addons:
   postgresql: "9.5"
 env:
   global:
-  - ETCDVERSION=3.0.8 ZKVERSION=3.4.9 CONSULVERSION=0.7.0
+  - ETCDVERSION=3.0.10 ZKVERSION=3.4.9 CONSULVERSION=0.7.0
   matrix:
   - TEST_SUITE="python setup.py"
   - DCS="etcd" TEST_SUITE="behave"
@@ -25,19 +25,22 @@ install:
     set -e
 
     if [[ $TEST_SUITE == "behave" ]]; then
-        if [[ $DCS == "consul" ]]; then
+        function get_consul() {
             curl -L https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip \
                 | gunzip > consul
+            [[ ${PIPESTATUS[0]} == 0 ]] || return 1
             chmod +x consul
-        fi
+        }
 
-        if [[ $DCS == "etcd" ]]; then
+        function get_etcd() {
             curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
                 | tar xz -C . --strip=1 --wildcards --no-anchored etcd
-        fi
+            return ${PIPESTATUS[0]}
+        }
 
-        if [[ $DCS == "exhibitor" ]]; then
+        function get_exhibitor() {
             curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz
+            [[ ${PIPESTATUS[0]} == 0 ]] || return 1
             mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg
             zookeeper-${ZKVERSION}/bin/zkServer.sh start
             # following lines are 'emulating' exhibitor REST API
@@ -45,7 +48,14 @@ install:
                 echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \
                     | nc -l 8181 &> /dev/null
             done&
-        fi
+        }
+
+        attempt_num=1
+        until get_${DCS}; do
+            [[ $attempt_num -ge 3 ]] && exit 1
+            echo "Attempt $attempt_num failed! Trying again in $attempt_num seconds..."
+            sleep $(( attempt_num++ ))
+        done
     fi
 
     for pv in "2.7" "3.4" "3.5"; do

features/environment.py

@@ -16,7 +16,8 @@ import yaml
 @six.add_metaclass(abc.ABCMeta)
 class AbstractController(object):
 
-    def __init__(self, name, work_directory, output_dir):
+    def __init__(self, context, name, work_directory, output_dir):
+        self._context = context
         self._name = name
         self._work_directory = work_directory
         self._output_dir = output_dir
@@ -46,6 +47,7 @@ class AbstractController(object):
 
         assert self._has_started(), "Process {0} is not running after being started".format(self._name)
 
+        max_wait_limit *= self._context.timeout_multiplier
         for _ in range(max_wait_limit):
             if self._is_accessible():
                 break
@@ -58,6 +60,7 @@ class AbstractController(object):
         term = False
         start_time = time.time()
 
+        timeout *= self._context.timeout_multiplier
         while self._handle and self._is_running():
             if kill:
                 self._handle.kill()
@@ -77,12 +80,12 @@ class PatroniController(AbstractController):
     PATRONI_CONFIG = '{}.yml'
     """ starts and stops individual patronis"""
 
-    def __init__(self, dcs, name, work_directory, output_dir, tags=None):
-        super(PatroniController, self).__init__('patroni_' + name, work_directory, output_dir)
+    def __init__(self, context, name, work_directory, output_dir, tags=None):
+        super(PatroniController, self).__init__(context, 'patroni_' + name, work_directory, output_dir)
         PatroniController.__PORT += 1
         self._data_dir = os.path.join(work_directory, 'data', name)
         self._connstring = None
-        self._config = self._make_patroni_test_config(name, dcs, tags)
+        self._config = self._make_patroni_test_config(name, tags)
 
         self._conn = None
         self._curs = None
@@ -112,7 +115,7 @@ class PatroniController(AbstractController):
     def _is_accessible(self):
         return self.query("SELECT 1", fail_ok=True) is not None
 
-    def _make_patroni_test_config(self, name, dcs, tags):
+    def _make_patroni_test_config(self, name, tags):
         patroni_config_name = self.PATRONI_CONFIG.format(name)
         patroni_config_path = os.path.join(self._output_dir, patroni_config_name)
 
@@ -182,6 +185,10 @@ class AbstractDcsController(AbstractController):
 
     _CLUSTER_NODE = '/service/batman'
 
+    def __init__(self, context, mktemp=True):
+        work_directory = mktemp and tempfile.mkdtemp() or None
+        super(AbstractDcsController, self).__init__(context, self.name(), work_directory, context.pctl.output_dir)
+
     def _is_accessible(self):
         return self._is_running()
 
@@ -206,11 +213,22 @@ class AbstractDcsController(AbstractController):
     def cleanup_service_tree(self):
         """ clean all contents stored in the tree used for the tests """
 
+    @classmethod
+    def get_subclasses(cls):
+        for subclass in cls.__subclasses__():
+            for subsubclass in subclass.get_subclasses():
+                yield subsubclass
+            yield subclass
+
+    @classmethod
+    def name(cls):
+        return cls.__name__[:-10].lower()
+
 
 class ConsulController(AbstractDcsController):
 
-    def __init__(self, output_dir):
-        super(ConsulController, self).__init__('consul', tempfile.mkdtemp(), output_dir)
+    def __init__(self, context):
+        super(ConsulController, self).__init__(context)
         os.environ['PATRONI_CONSUL_HOST'] = 'localhost:8500'
         self._client = consul.Consul()
 
@@ -245,8 +263,8 @@ class EtcdController(AbstractDcsController):
 
     """ handles all etcd related tasks, used for the tests setup and cleanup """
 
-    def __init__(self, output_dir):
-        super(EtcdController, self).__init__('etcd', tempfile.mkdtemp(), output_dir)
+    def __init__(self, context):
+        super(EtcdController, self).__init__(context)
         os.environ['PATRONI_ETCD_HOST'] = 'localhost:2379'
         self._client = etcd.Client(port=2379)
 
@@ -283,8 +301,8 @@ class ZooKeeperController(AbstractDcsController):
 
     """ handles all zookeeper related tasks, used for the tests setup and cleanup """
 
-    def __init__(self, output_dir, export_env=True):
-        super(ZooKeeperController, self).__init__('zookeeper', None, output_dir)
+    def __init__(self, context, export_env=True):
+        super(ZooKeeperController, self).__init__(context, False)
         if export_env:
             os.environ['PATRONI_ZOOKEEPER_HOSTS'] = "'localhost:2181'"
         self._client = kazoo.client.KazooClient()
@@ -321,22 +339,21 @@ class ZooKeeperController(AbstractDcsController):
 
 class ExhibitorController(ZooKeeperController):
 
-    def __init__(self, output_dir):
-        super(ExhibitorController, self).__init__(output_dir, False)
+    def __init__(self, context):
+        super(ExhibitorController, self).__init__(context, False)
         os.environ.update({'PATRONI_EXHIBITOR_HOSTS': 'localhost', 'PATRONI_EXHIBITOR_PORT': '8181'})
 
 
 class PatroniPoolController(object):
 
-    KNOWN_DCS = {'consul': ConsulController, 'etcd': EtcdController,
-                 'zookeeper': ZooKeeperController, 'exhibitor': ExhibitorController}
-
-    def __init__(self):
+    def __init__(self, context):
+        self._context = context
         self._dcs = None
         self._output_dir = None
         self._patroni_path = None
         self._processes = {}
         self.create_and_set_output_directory('')
+        self.known_dcs = {subclass.name(): subclass for subclass in AbstractDcsController.get_subclasses()}
 
     @property
     def patroni_path(self):
@@ -353,17 +370,17 @@ class PatroniPoolController(object):
     def output_dir(self):
         return self._output_dir
 
-    def start(self, pg_name, max_wait_limit=20, tags=None):
-        if pg_name not in self._processes:
-            self._processes[pg_name] = PatroniController(self.dcs, pg_name, self.patroni_path, self._output_dir, tags)
-        self._processes[pg_name].start(max_wait_limit)
+    def start(self, name, max_wait_limit=20, tags=None):
+        if name not in self._processes:
+            self._processes[name] = PatroniController(self._context, name, self.patroni_path, self._output_dir, tags)
+        self._processes[name].start(max_wait_limit)
 
     def __getattr__(self, func):
         if func not in ['stop', 'query', 'write_label', 'read_label', 'check_role_has_changed_to', 'add_tag_to_config']:
             raise AttributeError("PatroniPoolController instance has no attribute '{0}'".format(func))
 
-        def wrapper(pg_name, *args, **kwargs):
-            return getattr(self._processes[pg_name], func)(*args, **kwargs)
+        def wrapper(name, *args, **kwargs):
+            return getattr(self._processes[name], func)(*args, **kwargs)
         return wrapper
 
     def stop_all(self):
@@ -382,14 +399,15 @@ class PatroniPoolController(object):
     def dcs(self):
         if self._dcs is None:
             self._dcs = os.environ.pop('DCS', 'etcd')
-            assert self._dcs in self.KNOWN_DCS, 'Unsupported dcs: ' + self._dcs
+            assert self._dcs in self.known_dcs, 'Unsupported dcs: ' + self._dcs
         return self._dcs
 
 
 # actions to execute on start/stop of the tests and before running invidual features
 def before_all(context):
-    context.pctl = PatroniPoolController()
-    context.dcs_ctl = context.pctl.KNOWN_DCS[context.pctl.dcs](context.pctl.output_dir)
+    context.timeout_multiplier = 2 if 'TRAVIS_BUILD_NUMBER' in os.environ or 'BUILD_NUMBER' in os.environ else 1
+    context.pctl = PatroniPoolController(context)
+    context.dcs_ctl = context.pctl.known_dcs[context.pctl.dcs](context)
     context.dcs_ctl.start()
     try:
         context.dcs_ctl.cleanup_service_tree()

features/steps/basic_replication.py

@@ -30,6 +30,7 @@ def add_table(context, table_name, pg_name):
 
 @then('Table {table_name:w} is present on {pg_name:w} after {max_replication_delay:d} seconds')
 def table_is_present_on(context, table_name, pg_name, max_replication_delay):
+    max_replication_delay *= context.timeout_multiplier
     for _ in range(int(max_replication_delay)):
         if context.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
             break
@@ -41,6 +42,7 @@ def table_is_present_on(context, table_name, pg_name, max_replication_delay):
 
 @then('{pg_name:w} role is the {pg_role:w} after {max_promotion_timeout:d} seconds')
 def check_role(context, pg_name, pg_role, max_promotion_timeout):
+    max_promotion_timeout *= context.timeout_multiplier
     assert context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)),\
         "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
 

features/steps/cascading_replication.py

@@ -22,6 +22,7 @@ def write_label(context, content, name):
 
 @step('{name:w} has {key:w}={value:w} in dcs after {time_limit:d} seconds')
 def check_member(context, name, key, value, time_limit):
+    time_limit *= context.timeout_multiplier
     max_time = time.time() + int(time_limit)
     while time.time() < max_time:
         try:

features/steps/patroni_api.py

@@ -27,6 +27,7 @@ register_type(url=parse_url)
 @step('{name:w} is a leader after {time_limit:d} seconds')
 @then('{name:w} is a leader after {time_limit:d} seconds')
 def is_a_leader(context, name, time_limit):
+    time_limit *= context.timeout_multiplier
     max_time = time.time() + int(time_limit)
     while (context.dcs_ctl.query("leader") != name):
         time.sleep(1)
@@ -135,6 +136,7 @@ def add_tag_to_config(context, tag, value, pg_name):
 
 @then('Response on GET {url} contains {value} after {timeout:d} seconds')
 def check_http_response(context, url, value, timeout, negate=False):
+    timeout *= context.timeout_multiplier
     for _ in range(int(timeout)):
         r = requests.get(url)
         if (value in r.content.decode('utf-8')) != negate: