diff --git a/.github/workflows/install_deps.py b/.github/workflows/install_deps.py new file mode 100644 index 00000000..b795aeb2 --- /dev/null +++ b/.github/workflows/install_deps.py @@ -0,0 +1,187 @@ +import inspect +import os +import shutil +import subprocess +import stat +import sys +import tarfile +import time +import zipfile + + +def install_requirements(what): + old_path = sys.path[:] + w = os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe()))) + sys.path.insert(0, os.path.dirname(os.path.dirname(w))) + try: + from setup import EXTRAS_REQUIRE, read + finally: + sys.path = old_path + requirements = ['mock>=2.0.0', 'flake8', 'pytest', 'pytest-cov'] if what == 'all' else ['behave'] + requirements += ['psycopg2-binary', 'coverage'] + for r in read('requirements.txt').split('\n'): + r = r.strip() + if r != '': + extras = {e for e, v in EXTRAS_REQUIRE.items() if v and r.startswith(v[0])} + if not extras or what == 'all' or what in extras: + requirements.append(r) + + subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'pip']) + r = subprocess.call([sys.executable, '-m', 'pip', 'install'] + requirements) + s = subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'setuptools']) + return s | r + + +def install_packages(what): + packages = { + 'zookeeper': ['zookeeper', 'zookeeper-bin', 'zookeeperd'], + 'consul': ['consul'], + } + packages['exhibitor'] = packages['zookeeper'] + packages = packages.get(what, []) + ver = str({'etcd': '9.6', 'etcd3': '9.6', 'consul': 10, 'exhibitor': 11, 'kubernetes': 12, 'raft': 13}.get(what)) + return subprocess.call(['sudo', 'apt-get', 'install', '-y', 'postgresql-' + ver, 'expect-dev', 'wget'] + packages) + + +def get_file(url, name): + from six.moves.urllib.request import urlretrieve + print('Downloading ' + url) + urlretrieve(url, name) + + +def untar(archive, name): + with tarfile.open(archive) as tar: + f = tar.extractfile(name) + dest = os.path.basename(name) + with open(dest, 'wb') as d: + shutil.copyfileobj(f, d) + return dest + + +def unzip(archive, name): + with zipfile.ZipFile(archive, 'r') as z: + name = z.extract(name) + dest = os.path.basename(name) + shutil.move(name, dest) + return dest + + +def unzip_all(archive): + print('Extracting ' + archive) + with zipfile.ZipFile(archive, 'r') as z: + z.extractall() + + +def chmod_755(name): + os.chmod(name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) + + +def unpack(archive, name): + print('Extracting {0} from {1}'.format(name, archive)) + func = unzip if archive.endswith('.zip') else untar + name = func(archive, name) + chmod_755(name) + return name + + +def install_etcd(): + version = os.environ.get('ETCDVERSION', '3.3.13') + platform = {'linux2': 'linux', 'win32': 'windows', 'cygwin': 'windows'}.get(sys.platform, sys.platform) + dirname = 'etcd-v{0}-{1}-amd64'.format(version, platform) + ext = 'tar.gz' if platform == 'linux' else 'zip' + name = '{0}.{1}'.format(dirname, ext) + url = 'https://github.com/etcd-io/etcd/releases/download/v{0}/{1}'.format(version, name) + get_file(url, name) + ext = '.exe' if platform == 'windows' else '' + return int(unpack(name, '{0}/etcd{1}'.format(dirname, ext)) is None) + + +def install_postgres(): + version = os.environ.get('PGVERSION', '12.1-1') + platform = {'darwin': 'osx', 'win32': 'windows-x64', 'cygwin': 'windows-x64'}[sys.platform] + name = 'postgresql-{0}-{1}-binaries.zip'.format(version, platform) + get_file('http://get.enterprisedb.com/postgresql/' + name, name) + unzip_all(name) + bin_dir = os.path.join('pgsql', 'bin') + for f in os.listdir(bin_dir): + chmod_755(os.path.join(bin_dir, f)) + subprocess.call(['pgsql/bin/postgres', '-V']) + return 0 + + +def setup_kubernetes(): + get_file('https://storage.googleapis.com/minikube/k8sReleases/v1.7.0/localkube-linux-amd64', 'localkube') + chmod_755('localkube') + + devnull = open(os.devnull, 'w') + subprocess.Popen(['sudo', 'nohup', './localkube', '--logtostderr=true', '--enable-dns=false'], + stdout=devnull, stderr=devnull) + for _ in range(0, 120): + if subprocess.call(['wget', '-qO', '-', 'http://127.0.0.1:8080/'], stdout=devnull, stderr=devnull) == 0: + time.sleep(10) + break + time.sleep(1) + else: + print('localkube did not start') + return 1 + + subprocess.call('sudo chmod 644 /var/lib/localkube/certs/*', shell=True) + print('Set up .kube/config') + kube = os.path.join(os.path.expanduser('~'), '.kube') + os.makedirs(kube) + with open(os.path.join(kube, 'config'), 'w') as f: + f.write("""apiVersion: v1 +clusters: +- cluster: + certificate-authority: /var/lib/localkube/certs/ca.crt + server: https://127.0.0.1:8443 + name: local +contexts: +- context: + cluster: local + user: myself + name: local +current-context: local +kind: Config +preferences: {} +users: +- name: myself + user: + client-certificate: /var/lib/localkube/certs/apiserver.crt + client-key: /var/lib/localkube/certs/apiserver.key +""") + return 0 + + +def setup_exhibitor(): + response = '{"servers":["127.0.0.1"],"port":2181}' + response = 'HTTP/1.0 200 OK\\nContent-Length: {0}\\n\\n{1}'.format(len(response), response) + s = subprocess.Popen("while true; do echo '{0}'| nc -l 8181 > /dev/null; done".format(response), shell=True) + return 0 if s.poll() is None else s.returncode + + +def main(): + what = os.environ.get('DCS', sys.argv[1] if len(sys.argv) > 1 else 'all') + r = install_requirements(what) + if what == 'all' or r != 0: + return r + + if sys.platform.startswith('linux'): + r = install_packages(what) + else: + r = install_postgres() + if r != 0: + return r + + if what.startswith('etcd'): + return install_etcd() + elif what == 'kubernetes': + return setup_kubernetes() + elif what == 'exhibitor': + return setup_exhibitor() + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/workflows/run_tests.py b/.github/workflows/run_tests.py new file mode 100644 index 00000000..e2db5e93 --- /dev/null +++ b/.github/workflows/run_tests.py @@ -0,0 +1,47 @@ +import os +import shutil +import subprocess +import sys +import tempfile + + +def main(): + what = os.environ.get('DCS', sys.argv[1] if len(sys.argv) > 1 else 'all') + + if what == 'all': + flake8 = subprocess.call([sys.executable, 'setup.py', 'flake8']) + test = subprocess.call([sys.executable, 'setup.py', 'test']) + version = '.'.join(map(str, sys.version_info[:2])) + shutil.move('.coverage', os.path.join(tempfile.gettempdir(), '.coverage.' + version)) + return flake8 | test + elif what == 'combine': + tmp = tempfile.gettempdir() + for name in os.listdir(tmp): + if name.startswith('.coverage.'): + shutil.move(os.path.join(tmp, name), name) + return subprocess.call([sys.executable, '-m', 'coverage', 'combine']) + + env = os.environ.copy() + if sys.platform.startswith('linux'): + version = {'etcd': '9.6', 'etcd3': '9.6', 'consul': 10, 'exhibitor': 11, 'kubernetes': 12, 'raft': 13}.get(what) + path = '/usr/lib/postgresql/{0}/bin:.'.format(version) + unbuffer = ['timeout', '600', 'unbuffer'] + else: + path = os.path.abspath(os.path.join('pgsql', 'bin')) + if sys.platform == 'darwin': + path += ':.' + unbuffer = [] + env['PATH'] = path + os.pathsep + env['PATH'] + env['DCS'] = what + + ret = subprocess.call(unbuffer + [sys.executable, '-m', 'behave'], env=env) + + if ret != 0: + if subprocess.call('grep . features/output/*_failed/*postgres?.*', shell=True) != 0: + subprocess.call('grep . features/output/*/*postgres?.*', shell=True) + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 00000000..1595a214 --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,173 @@ +name: Tests + +on: + pull_request: + push: + branches: + - master + tags: + - v.* + +jobs: + unit: + runs-on: ${{ matrix.os }}-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu, windows, macos] + + steps: + - uses: actions/checkout@v1 + - name: Set up Python 2.7 + uses: actions/setup-python@v2 + with: + python-version: 2.7 + if: matrix.os != 'windows' + - name: Install dependencies + run: python .github/workflows/install_deps.py + if: matrix.os != 'windows' + - name: Run tests and flake8 + run: python .github/workflows/run_tests.py + if: matrix.os != 'windows' + + - name: Set up Python 3.5 + uses: actions/setup-python@v2 + with: + python-version: 3.5 + - name: Install dependencies + run: python .github/workflows/install_deps.py + - name: Run tests and flake8 + run: python .github/workflows/run_tests.py + + - name: Set up Python 3.6 + uses: actions/setup-python@v2 + with: + python-version: 3.6 + - name: Install dependencies + run: python .github/workflows/install_deps.py + - name: Run tests and flake8 + run: python .github/workflows/run_tests.py + + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Install dependencies + run: python .github/workflows/install_deps.py + - name: Run tests and flake8 + run: python .github/workflows/run_tests.py + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies + run: python .github/workflows/install_deps.py + - name: Run tests and flake8 + run: python .github/workflows/run_tests.py + + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install dependencies + run: python .github/workflows/install_deps.py + - name: Run tests and flake8 + run: python .github/workflows/run_tests.py + + - name: Combine coverage + run: python .github/workflows/run_tests.py combine + + - name: Install coveralls + run: python -m pip install coveralls + + - name: Upload Coverage + env: + COVERALLS_FLAG_NAME: unit-${{ matrix.os }} + COVERALLS_PARALLEL: 'true' + GITHUB_TOKEN: ${{ secrets.github_token }} + run: python -m coveralls + + - name: Run codacy-coverage-reporter + uses: codacy/codacy-coverage-reporter-action@master + with: + project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} + coverage-reports: coverage.xml + if: matrix.os == 'ubuntu' + + behave: + runs-on: ${{ matrix.os }}-latest + env: + DCS: ${{ matrix.dcs }} + ETCDVERSION: 3.3.13 + strategy: + fail-fast: false + matrix: + os: [ubuntu] + python-version: [2.7, 3.5, 3.8] + dcs: [etcd, etcd3, consul, exhibitor, kubernetes, raft] + + steps: + - uses: actions/checkout@v1 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: python .github/workflows/install_deps.py + - name: Run behave tests + run: python .github/workflows/run_tests.py + - uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install coveralls + run: python -m pip install coveralls + - name: Upload Coverage + env: + COVERALLS_FLAG_NAME: behave-${{ matrix.os }}-${{ matrix.dcs }}-${{ matrix.python-version }} + COVERALLS_PARALLEL: 'true' + GITHUB_TOKEN: ${{ secrets.github_token }} + run: python -m coveralls + + behavem: + runs-on: ${{ matrix.os }}-latest + env: + DCS: ${{ matrix.dcs }} + ETCDVERSION: 3.3.13 + PGVERSION: 12.1-1 # for windows and macos + strategy: + fail-fast: false + matrix: + os: [macos] #, windows] + python-version: [3.7] + dcs: [etcd, etcd3, raft] + + steps: + - uses: actions/checkout@v1 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: python .github/workflows/install_deps.py + - name: Run behave tests + run: python .github/workflows/run_tests.py + - name: Install coveralls + run: python -m pip install coveralls + - name: Upload Coverage + env: + COVERALLS_FLAG_NAME: behave-${{ matrix.os }}-${{ matrix.dcs }}-${{ matrix.python-version }} + COVERALLS_PARALLEL: 'true' + GITHUB_TOKEN: ${{ secrets.github_token }} + run: python -m coveralls + + coveralls-finish: + name: Finalize coveralls.io + needs: [unit, behave, behavem] + runs-on: ubuntu-latest + steps: + - uses: actions/setup-python@v2 + - run: python -m pip install coveralls + - run: python -m coveralls --finish + env: + GITHUB_TOKEN: ${{ secrets.github_token }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index de1bc6eb..00000000 --- a/.travis.yml +++ /dev/null @@ -1,165 +0,0 @@ -sudo: true -dist: trusty -language: python -addons: - apt: - packages: - - expect-dev # for unbuffer -env: - global: - - ETCDVERSION=3.3.13 ZKVERSION=3.6.2 CONSULVERSION=0.7.4 - - PYVERSIONS="2.7 3.6" - - BOTO_CONFIG=/doesnotexist -matrix: - include: - - python: "3.5" - env: PYVERSIONS="2.7 3.5 3.6" TEST_SUITE="python setup.py" - - python: "3.6" - env: DCS="etcd" TEST_SUITE="behave" - - python: "3.6" - env: DCS="etcd3" TEST_SUITE="behave" - - python: "3.6" - env: DCS="exhibitor" TEST_SUITE="behave" - - python: "3.6" - env: DCS="consul" TEST_SUITE="behave" - - python: "3.6" - env: DCS="raft" TEST_SUITE="behave" - - python: "3.6" - env: DCS="kubernetes" TEST_SUITE="behave" -branches: - only: - - master - - /^v\d+\.\d+(\.\d+)?$/ -cache: - directories: - - $HOME/mycache -before_cache: -- | - rm -fr $HOME/mycache/python* - for pv in $PYVERSIONS; do - fpv=$(basename $(readlink $HOME/virtualenv/python${pv})) - mv $HOME/virtualenv/${fpv} $HOME/mycache/${fpv} - done -install: - - | - set -e - - if [[ $TEST_SUITE == "behave" ]]; then - function get_consul() { - CC=~/mycache/consul_${CONSULVERSION} - if [[ ! -x $CC ]]; then - curl -L https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip \ - | gunzip > $CC - [[ ${PIPESTATUS[0]} == 0 ]] || return 1 - chmod +x $CC - fi - ln -s $CC consul - } - - function get_etcd() { - EC=~/mycache/etcd_${ETCDVERSION} - if [[ ! -x $EC ]]; then - rm -rf ~/mycache/etcd_* - curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \ - | tar xz -C . --strip=1 --wildcards --no-anchored etcd - [[ ${PIPESTATUS[0]} == 0 ]] || return 1 - mv etcd $EC - fi - ln -s $EC etcd - } - - function get_etcd3() { - get_etcd - } - - function get_kubernetes() { - wget -O localkube "https://storage.googleapis.com/minikube/k8sReleases/v1.7.0/localkube-linux-amd64" - chmod +x localkube - sudo nohup ./localkube --logtostderr=true --enable-dns=false > localkube.log 2>&1 & - - echo "Waiting for localkube to start..." - if ! timeout 120 sh -c "while ! curl -ks http://127.0.0.1:8080/ >/dev/null; do sleep 1; done"; then - sudo cat localkube.log - echo "localkube did not start" - exit 1 - fi - echo "Check certificate permissions" - sudo chmod 644 /var/lib/localkube/certs/* - sudo ls -altr /var/lib/localkube/certs/ - - echo "Set up .kube/config" - mkdir ~/.kube - echo -e "apiVersion: v1\nclusters:\n- cluster:\n certificate-authority: /var/lib/localkube/certs/ca.crt\n server: https://127.0.0.1:8443\n name: local\ncontexts:\n- context:\n cluster: local\n user: myself\n name: local\ncurrent-context: local\nkind: Config\npreferences: {}\nusers:\n- name: myself\n user:\n client-certificate: /var/lib/localkube/certs/apiserver.crt\n client-key: /var/lib/localkube/certs/apiserver.key\n" > ~/.kube/config - } - - function get_exhibitor() { - ZK_PACKAGE=apache-zookeeper-${ZKVERSION}-bin - ZK=~/mycache/$ZK_PACKAGE - if [[ ! -d $ZK ]]; then - curl -L https://downloads.apache.org/zookeeper/zookeeper-${ZKVERSION}/${ZK_PACKAGE}.tar.gz | tar xz - [[ ${PIPESTATUS[0]} == 0 ]] || return 1 - mv $ZK_PACKAGE/conf/zoo_sample.cfg $ZK_PACKAGE/conf/zoo.cfg - mv $ZK_PACKAGE $ZK - fi - $ZK/bin/zkServer.sh start - # following lines are 'emulating' exhibitor REST API - while true; do - echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \ - | nc -l 8181 &> /dev/null - done& - } - - function get_raft() { - return 0 - } - - attempt_num=1 - until get_${DCS}; do - [[ $attempt_num -ge 3 ]] && exit 1 - echo "Attempt $attempt_num failed! Trying again in $attempt_num seconds..." - sleep $(( attempt_num++ )) - done - fi - - for pv in $PYVERSIONS; do - fpv=$(basename $(readlink $HOME/virtualenv/python$pv)) - if [[ -d ~/mycache/${fpv} ]]; then - mv ~/virtualenv/${fpv} ~/virtualenv/${fpv}.bckp - mv ~/mycache/${fpv} ~/virtualenv/${fpv} - fi - source ~/virtualenv/python${pv}/bin/activate - # explicitly install all needed python modules to cache them - for p in '-r requirements.txt' '-r requirements.dev.txt' 'codacy-coverage coveralls'; do - pip install $p --upgrade - done - done -script: - - | - for pv in $PYVERSIONS; do - source ~/virtualenv/python${pv}/bin/activate - - if [[ $TEST_SUITE = "behave" ]]; then - echo Running integration tests using python${pv} - if ! PATH=.:/usr/lib/postgresql/9.6/bin:$PATH unbuffer $TEST_SUITE; then - # output all log files when tests are failing - grep . features/output/*_failed/*postgres?.* - exit 1 - fi - else - echo Running unit tests using python${pv} - unbuffer $TEST_SUITE test - $TEST_SUITE flake8 - fi - mv .coverage /tmp/.coverage.$pv - done - mv /tmp/.coverage.* . - python -m coverage combine - - set +e -after_success: - # before_cache is executed earlier than after_success, so we need to restore one of virtualenv directories - - fpv=$(basename $(readlink $HOME/virtualenv/python3.6)) && mv $HOME/mycache/${fpv} $HOME/virtualenv/${fpv} - - coveralls - - if [[ $TEST_SUITE != "behave" ]]; then python-codacy-coverage -r coverage.xml; fi - - if [[ $DCS == "exhibitor" ]]; then ~/mycache/zookeeper-${ZKVERSION}/bin/zkServer.sh stop; fi - - sudo kill $(jobs -p) diff --git a/docs/CONTRIBUTING.rst b/docs/CONTRIBUTING.rst index fd46fabf..b53ead20 100644 --- a/docs/CONTRIBUTING.rst +++ b/docs/CONTRIBUTING.rst @@ -17,7 +17,7 @@ Requirements for running behave tests: 1. PostgreSQL packages need to be installed. 2. PostgreSQL binaries must be available in your `PATH`. You may need to add them to the path with something like `PATH=/usr/lib/postgresql/11/bin:$PATH python -m behave`. -3. If you'd like to test with external DCSs (e.g., Etcd, Consul, and Zookeeper) you'll need the packages installed and respective services running and accepting unencrypted/unprotected connections on localhost and default port. In the case of Etcd or Consul, the behave test suite could start them up if binaries are available in the `PATH`. See the `get_` functions in the scripting in [the Travis configuration](.travis.yml) for more details. +3. If you'd like to test with external DCSs (e.g., Etcd, Consul, and Zookeeper) you'll need the packages installed and respective services running and accepting unencrypted/unprotected connections on localhost and default port. In the case of Etcd or Consul, the behave test suite could start them up if binaries are available in the `PATH`. Install dependencies: diff --git a/features/environment.py b/features/environment.py index e04399c6..b2ff6c86 100644 --- a/features/environment.py +++ b/features/environment.py @@ -176,6 +176,7 @@ class PatroniController(AbstractController): config['name'] = name config['postgresql']['data_dir'] = self._data_dir + config['postgresql']['basebackup'] = [{'checkpoint': 'fast'}] config['postgresql']['use_unix_socket'] = os.name != 'nt' # windows doesn't yet support unix-domain sockets config['postgresql']['pgpass'] = os.path.join(tempfile.gettempdir(), 'pgpass_' + name) config['postgresql']['parameters'].update({ @@ -849,8 +850,8 @@ class WatchdogMonitor(object): # actions to execute on start/stop of the tests and before running invidual features def before_all(context): os.environ.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'}) - context.ci = 'TRAVIS_BUILD_NUMBER' in os.environ or 'BUILD_NUMBER' in os.environ - context.timeout_multiplier = 2 if context.ci else 1 + context.ci = any(a in os.environ for a in ('TRAVIS_BUILD_NUMBER', 'BUILD_NUMBER', 'GITHUB_ACTIONS')) + context.timeout_multiplier = 5 if context.ci else 1 # MacOS sometimes is VERY slow context.pctl = PatroniPoolController(context) context.dcs_ctl = context.pctl.known_dcs[context.pctl.dcs](context) context.dcs_ctl.start()