From 4594bc98da0009c9fe04caa8642369fad9c70829 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 28 Sep 2016 15:13:09 +0200 Subject: [PATCH] Increase timeouts when running AT on travis (#324) * Increase timeouts two times when running AT on travis * Make up to 3 attempts to download DCS * Get rid from hard-coded names --- .travis.yml | 24 ++++++--- features/environment.py | 70 ++++++++++++++++--------- features/steps/basic_replication.py | 2 + features/steps/cascading_replication.py | 1 + features/steps/patroni_api.py | 2 + 5 files changed, 66 insertions(+), 33 deletions(-) diff --git a/.travis.yml b/.travis.yml index 20c9273b..e88b83c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ addons: postgresql: "9.5" env: global: - - ETCDVERSION=3.0.8 ZKVERSION=3.4.9 CONSULVERSION=0.7.0 + - ETCDVERSION=3.0.10 ZKVERSION=3.4.9 CONSULVERSION=0.7.0 matrix: - TEST_SUITE="python setup.py" - DCS="etcd" TEST_SUITE="behave" @@ -25,19 +25,22 @@ install: set -e if [[ $TEST_SUITE == "behave" ]]; then - if [[ $DCS == "consul" ]]; then + function get_consul() { curl -L https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip \ | gunzip > consul + [[ ${PIPESTATUS[0]} == 0 ]] || return 1 chmod +x consul - fi + } - if [[ $DCS == "etcd" ]]; then + function get_etcd() { curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \ | tar xz -C . --strip=1 --wildcards --no-anchored etcd - fi + return ${PIPESTATUS[0]} + } - if [[ $DCS == "exhibitor" ]]; then + function get_exhibitor() { curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz + [[ ${PIPESTATUS[0]} == 0 ]] || return 1 mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg zookeeper-${ZKVERSION}/bin/zkServer.sh start # following lines are 'emulating' exhibitor REST API @@ -45,7 +48,14 @@ install: echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \ | nc -l 8181 &> /dev/null done& - fi + } + + attempt_num=1 + until get_${DCS}; do + [[ $attempt_num -ge 3 ]] && exit 1 + echo "Attempt $attempt_num failed! Trying again in $attempt_num seconds..." + sleep $(( attempt_num++ )) + done fi for pv in "2.7" "3.4" "3.5"; do diff --git a/features/environment.py b/features/environment.py index 3c955e5d..8fe41064 100644 --- a/features/environment.py +++ b/features/environment.py @@ -16,7 +16,8 @@ import yaml @six.add_metaclass(abc.ABCMeta) class AbstractController(object): - def __init__(self, name, work_directory, output_dir): + def __init__(self, context, name, work_directory, output_dir): + self._context = context self._name = name self._work_directory = work_directory self._output_dir = output_dir @@ -46,6 +47,7 @@ class AbstractController(object): assert self._has_started(), "Process {0} is not running after being started".format(self._name) + max_wait_limit *= self._context.timeout_multiplier for _ in range(max_wait_limit): if self._is_accessible(): break @@ -58,6 +60,7 @@ class AbstractController(object): term = False start_time = time.time() + timeout *= self._context.timeout_multiplier while self._handle and self._is_running(): if kill: self._handle.kill() @@ -77,12 +80,12 @@ class PatroniController(AbstractController): PATRONI_CONFIG = '{}.yml' """ starts and stops individual patronis""" - def __init__(self, dcs, name, work_directory, output_dir, tags=None): - super(PatroniController, self).__init__('patroni_' + name, work_directory, output_dir) + def __init__(self, context, name, work_directory, output_dir, tags=None): + super(PatroniController, self).__init__(context, 'patroni_' + name, work_directory, output_dir) PatroniController.__PORT += 1 self._data_dir = os.path.join(work_directory, 'data', name) self._connstring = None - self._config = self._make_patroni_test_config(name, dcs, tags) + self._config = self._make_patroni_test_config(name, tags) self._conn = None self._curs = None @@ -112,7 +115,7 @@ class PatroniController(AbstractController): def _is_accessible(self): return self.query("SELECT 1", fail_ok=True) is not None - def _make_patroni_test_config(self, name, dcs, tags): + def _make_patroni_test_config(self, name, tags): patroni_config_name = self.PATRONI_CONFIG.format(name) patroni_config_path = os.path.join(self._output_dir, patroni_config_name) @@ -182,6 +185,10 @@ class AbstractDcsController(AbstractController): _CLUSTER_NODE = '/service/batman' + def __init__(self, context, mktemp=True): + work_directory = mktemp and tempfile.mkdtemp() or None + super(AbstractDcsController, self).__init__(context, self.name(), work_directory, context.pctl.output_dir) + def _is_accessible(self): return self._is_running() @@ -206,11 +213,22 @@ class AbstractDcsController(AbstractController): def cleanup_service_tree(self): """ clean all contents stored in the tree used for the tests """ + @classmethod + def get_subclasses(cls): + for subclass in cls.__subclasses__(): + for subsubclass in subclass.get_subclasses(): + yield subsubclass + yield subclass + + @classmethod + def name(cls): + return cls.__name__[:-10].lower() + class ConsulController(AbstractDcsController): - def __init__(self, output_dir): - super(ConsulController, self).__init__('consul', tempfile.mkdtemp(), output_dir) + def __init__(self, context): + super(ConsulController, self).__init__(context) os.environ['PATRONI_CONSUL_HOST'] = 'localhost:8500' self._client = consul.Consul() @@ -245,8 +263,8 @@ class EtcdController(AbstractDcsController): """ handles all etcd related tasks, used for the tests setup and cleanup """ - def __init__(self, output_dir): - super(EtcdController, self).__init__('etcd', tempfile.mkdtemp(), output_dir) + def __init__(self, context): + super(EtcdController, self).__init__(context) os.environ['PATRONI_ETCD_HOST'] = 'localhost:2379' self._client = etcd.Client(port=2379) @@ -283,8 +301,8 @@ class ZooKeeperController(AbstractDcsController): """ handles all zookeeper related tasks, used for the tests setup and cleanup """ - def __init__(self, output_dir, export_env=True): - super(ZooKeeperController, self).__init__('zookeeper', None, output_dir) + def __init__(self, context, export_env=True): + super(ZooKeeperController, self).__init__(context, False) if export_env: os.environ['PATRONI_ZOOKEEPER_HOSTS'] = "'localhost:2181'" self._client = kazoo.client.KazooClient() @@ -321,22 +339,21 @@ class ZooKeeperController(AbstractDcsController): class ExhibitorController(ZooKeeperController): - def __init__(self, output_dir): - super(ExhibitorController, self).__init__(output_dir, False) + def __init__(self, context): + super(ExhibitorController, self).__init__(context, False) os.environ.update({'PATRONI_EXHIBITOR_HOSTS': 'localhost', 'PATRONI_EXHIBITOR_PORT': '8181'}) class PatroniPoolController(object): - KNOWN_DCS = {'consul': ConsulController, 'etcd': EtcdController, - 'zookeeper': ZooKeeperController, 'exhibitor': ExhibitorController} - - def __init__(self): + def __init__(self, context): + self._context = context self._dcs = None self._output_dir = None self._patroni_path = None self._processes = {} self.create_and_set_output_directory('') + self.known_dcs = {subclass.name(): subclass for subclass in AbstractDcsController.get_subclasses()} @property def patroni_path(self): @@ -353,17 +370,17 @@ class PatroniPoolController(object): def output_dir(self): return self._output_dir - def start(self, pg_name, max_wait_limit=20, tags=None): - if pg_name not in self._processes: - self._processes[pg_name] = PatroniController(self.dcs, pg_name, self.patroni_path, self._output_dir, tags) - self._processes[pg_name].start(max_wait_limit) + def start(self, name, max_wait_limit=20, tags=None): + if name not in self._processes: + self._processes[name] = PatroniController(self._context, name, self.patroni_path, self._output_dir, tags) + self._processes[name].start(max_wait_limit) def __getattr__(self, func): if func not in ['stop', 'query', 'write_label', 'read_label', 'check_role_has_changed_to', 'add_tag_to_config']: raise AttributeError("PatroniPoolController instance has no attribute '{0}'".format(func)) - def wrapper(pg_name, *args, **kwargs): - return getattr(self._processes[pg_name], func)(*args, **kwargs) + def wrapper(name, *args, **kwargs): + return getattr(self._processes[name], func)(*args, **kwargs) return wrapper def stop_all(self): @@ -382,14 +399,15 @@ class PatroniPoolController(object): def dcs(self): if self._dcs is None: self._dcs = os.environ.pop('DCS', 'etcd') - assert self._dcs in self.KNOWN_DCS, 'Unsupported dcs: ' + self._dcs + assert self._dcs in self.known_dcs, 'Unsupported dcs: ' + self._dcs return self._dcs # actions to execute on start/stop of the tests and before running invidual features def before_all(context): - context.pctl = PatroniPoolController() - context.dcs_ctl = context.pctl.KNOWN_DCS[context.pctl.dcs](context.pctl.output_dir) + context.timeout_multiplier = 2 if 'TRAVIS_BUILD_NUMBER' in os.environ or 'BUILD_NUMBER' in os.environ else 1 + context.pctl = PatroniPoolController(context) + context.dcs_ctl = context.pctl.known_dcs[context.pctl.dcs](context) context.dcs_ctl.start() try: context.dcs_ctl.cleanup_service_tree() diff --git a/features/steps/basic_replication.py b/features/steps/basic_replication.py index b59a7639..7360d4a0 100644 --- a/features/steps/basic_replication.py +++ b/features/steps/basic_replication.py @@ -30,6 +30,7 @@ def add_table(context, table_name, pg_name): @then('Table {table_name:w} is present on {pg_name:w} after {max_replication_delay:d} seconds') def table_is_present_on(context, table_name, pg_name, max_replication_delay): + max_replication_delay *= context.timeout_multiplier for _ in range(int(max_replication_delay)): if context.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None: break @@ -41,6 +42,7 @@ def table_is_present_on(context, table_name, pg_name, max_replication_delay): @then('{pg_name:w} role is the {pg_role:w} after {max_promotion_timeout:d} seconds') def check_role(context, pg_name, pg_role, max_promotion_timeout): + max_promotion_timeout *= context.timeout_multiplier assert context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)),\ "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout) diff --git a/features/steps/cascading_replication.py b/features/steps/cascading_replication.py index b80ae659..ba0a6f7d 100644 --- a/features/steps/cascading_replication.py +++ b/features/steps/cascading_replication.py @@ -22,6 +22,7 @@ def write_label(context, content, name): @step('{name:w} has {key:w}={value:w} in dcs after {time_limit:d} seconds') def check_member(context, name, key, value, time_limit): + time_limit *= context.timeout_multiplier max_time = time.time() + int(time_limit) while time.time() < max_time: try: diff --git a/features/steps/patroni_api.py b/features/steps/patroni_api.py index fc4466e8..9e055956 100644 --- a/features/steps/patroni_api.py +++ b/features/steps/patroni_api.py @@ -27,6 +27,7 @@ register_type(url=parse_url) @step('{name:w} is a leader after {time_limit:d} seconds') @then('{name:w} is a leader after {time_limit:d} seconds') def is_a_leader(context, name, time_limit): + time_limit *= context.timeout_multiplier max_time = time.time() + int(time_limit) while (context.dcs_ctl.query("leader") != name): time.sleep(1) @@ -135,6 +136,7 @@ def add_tag_to_config(context, tag, value, pg_name): @then('Response on GET {url} contains {value} after {timeout:d} seconds') def check_http_response(context, url, value, timeout, negate=False): + timeout *= context.timeout_multiplier for _ in range(int(timeout)): r = requests.get(url) if (value in r.content.decode('utf-8')) != negate: