Switch to GH actions (#1778)

it allows up to 20 parallel builds
2026-08-25 14:53:37 +00:00 · 2020-12-04 21:52:34 +01:00
parent e2b2daf0b3
commit 1530ed0b9c
6 changed files with 411 additions and 168 deletions
@@ -0,0 +1,187 @@
+import inspect
+import os
+import shutil
+import subprocess
+import stat
+import sys
+import tarfile
+import time
+import zipfile
+
+
+def install_requirements(what):
+    old_path = sys.path[:]
+    w = os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe())))
+    sys.path.insert(0, os.path.dirname(os.path.dirname(w)))
+    try:
+        from setup import EXTRAS_REQUIRE, read
+    finally:
+        sys.path = old_path
+    requirements = ['mock>=2.0.0', 'flake8', 'pytest', 'pytest-cov'] if what == 'all' else ['behave']
+    requirements += ['psycopg2-binary', 'coverage']
+    for r in read('requirements.txt').split('\n'):
+        r = r.strip()
+        if r != '':
+            extras = {e for e, v in EXTRAS_REQUIRE.items() if v and r.startswith(v[0])}
+            if not extras or what == 'all' or what in extras:
+                requirements.append(r)
+
+    subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'pip'])
+    r = subprocess.call([sys.executable, '-m', 'pip', 'install'] + requirements)
+    s = subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'setuptools'])
+    return s | r
+
+
+def install_packages(what):
+    packages = {
+        'zookeeper': ['zookeeper', 'zookeeper-bin', 'zookeeperd'],
+        'consul': ['consul'],
+    }
+    packages['exhibitor'] = packages['zookeeper']
+    packages = packages.get(what, [])
+    ver = str({'etcd': '9.6', 'etcd3': '9.6', 'consul': 10, 'exhibitor': 11, 'kubernetes': 12, 'raft': 13}.get(what))
+    return subprocess.call(['sudo', 'apt-get', 'install', '-y', 'postgresql-' + ver, 'expect-dev', 'wget'] + packages)
+
+
+def get_file(url, name):
+    from six.moves.urllib.request import urlretrieve
+    print('Downloading ' + url)
+    urlretrieve(url, name)
+
+
+def untar(archive, name):
+    with tarfile.open(archive) as tar:
+        f = tar.extractfile(name)
+        dest = os.path.basename(name)
+        with open(dest, 'wb') as d:
+            shutil.copyfileobj(f, d)
+            return dest
+
+
+def unzip(archive, name):
+    with zipfile.ZipFile(archive, 'r') as z:
+        name = z.extract(name)
+        dest = os.path.basename(name)
+        shutil.move(name, dest)
+        return dest
+
+
+def unzip_all(archive):
+    print('Extracting ' + archive)
+    with zipfile.ZipFile(archive, 'r') as z:
+        z.extractall()
+
+
+def chmod_755(name):
+    os.chmod(name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
+             stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
+
+
+def unpack(archive, name):
+    print('Extracting {0} from {1}'.format(name, archive))
+    func = unzip if archive.endswith('.zip') else untar
+    name = func(archive, name)
+    chmod_755(name)
+    return name
+
+
+def install_etcd():
+    version = os.environ.get('ETCDVERSION', '3.3.13')
+    platform = {'linux2': 'linux', 'win32': 'windows', 'cygwin': 'windows'}.get(sys.platform, sys.platform)
+    dirname = 'etcd-v{0}-{1}-amd64'.format(version, platform)
+    ext = 'tar.gz' if platform == 'linux' else 'zip'
+    name = '{0}.{1}'.format(dirname, ext)
+    url = 'https://github.com/etcd-io/etcd/releases/download/v{0}/{1}'.format(version, name)
+    get_file(url, name)
+    ext = '.exe' if platform == 'windows' else ''
+    return int(unpack(name, '{0}/etcd{1}'.format(dirname, ext)) is None)
+
+
+def install_postgres():
+    version = os.environ.get('PGVERSION', '12.1-1')
+    platform = {'darwin': 'osx', 'win32': 'windows-x64', 'cygwin': 'windows-x64'}[sys.platform]
+    name = 'postgresql-{0}-{1}-binaries.zip'.format(version, platform)
+    get_file('http://get.enterprisedb.com/postgresql/' + name, name)
+    unzip_all(name)
+    bin_dir = os.path.join('pgsql', 'bin')
+    for f in os.listdir(bin_dir):
+        chmod_755(os.path.join(bin_dir, f))
+    subprocess.call(['pgsql/bin/postgres', '-V'])
+    return 0
+
+
+def setup_kubernetes():
+    get_file('https://storage.googleapis.com/minikube/k8sReleases/v1.7.0/localkube-linux-amd64', 'localkube')
+    chmod_755('localkube')
+
+    devnull = open(os.devnull, 'w')
+    subprocess.Popen(['sudo', 'nohup', './localkube', '--logtostderr=true', '--enable-dns=false'],
+                     stdout=devnull, stderr=devnull)
+    for _ in range(0, 120):
+        if subprocess.call(['wget', '-qO', '-', 'http://127.0.0.1:8080/'], stdout=devnull, stderr=devnull) == 0:
+            time.sleep(10)
+            break
+        time.sleep(1)
+    else:
+        print('localkube did not start')
+        return 1
+
+    subprocess.call('sudo chmod 644 /var/lib/localkube/certs/*', shell=True)
+    print('Set up .kube/config')
+    kube = os.path.join(os.path.expanduser('~'), '.kube')
+    os.makedirs(kube)
+    with open(os.path.join(kube, 'config'), 'w') as f:
+        f.write("""apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority: /var/lib/localkube/certs/ca.crt
+    server: https://127.0.0.1:8443
+  name: local
+contexts:
+- context:
+    cluster: local
+    user: myself
+  name: local
+current-context: local
+kind: Config
+preferences: {}
+users:
+- name: myself
+  user:
+    client-certificate: /var/lib/localkube/certs/apiserver.crt
+    client-key: /var/lib/localkube/certs/apiserver.key
+""")
+    return 0
+
+
+def setup_exhibitor():
+    response = '{"servers":["127.0.0.1"],"port":2181}'
+    response = 'HTTP/1.0 200 OK\\nContent-Length: {0}\\n\\n{1}'.format(len(response), response)
+    s = subprocess.Popen("while true; do echo '{0}'| nc -l 8181 > /dev/null; done".format(response), shell=True)
+    return 0 if s.poll() is None else s.returncode
+
+
+def main():
+    what = os.environ.get('DCS', sys.argv[1] if len(sys.argv) > 1 else 'all')
+    r = install_requirements(what)
+    if what == 'all' or r != 0:
+        return r
+
+    if sys.platform.startswith('linux'):
+        r = install_packages(what)
+    else:
+        r = install_postgres()
+    if r != 0:
+        return r
+
+    if what.startswith('etcd'):
+        return install_etcd()
+    elif what == 'kubernetes':
+        return setup_kubernetes()
+    elif what == 'exhibitor':
+        return setup_exhibitor()
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
@@ -0,0 +1,47 @@
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def main():
+    what = os.environ.get('DCS', sys.argv[1] if len(sys.argv) > 1 else 'all')
+
+    if what == 'all':
+        flake8 = subprocess.call([sys.executable, 'setup.py', 'flake8'])
+        test = subprocess.call([sys.executable, 'setup.py', 'test'])
+        version = '.'.join(map(str, sys.version_info[:2]))
+        shutil.move('.coverage', os.path.join(tempfile.gettempdir(), '.coverage.' + version))
+        return flake8 | test
+    elif what == 'combine':
+        tmp = tempfile.gettempdir()
+        for name in os.listdir(tmp):
+            if name.startswith('.coverage.'):
+                shutil.move(os.path.join(tmp, name), name)
+        return subprocess.call([sys.executable, '-m', 'coverage', 'combine'])
+
+    env = os.environ.copy()
+    if sys.platform.startswith('linux'):
+        version = {'etcd': '9.6', 'etcd3': '9.6', 'consul': 10, 'exhibitor': 11, 'kubernetes': 12, 'raft': 13}.get(what)
+        path = '/usr/lib/postgresql/{0}/bin:.'.format(version)
+        unbuffer = ['timeout', '600', 'unbuffer']
+    else:
+        path = os.path.abspath(os.path.join('pgsql', 'bin'))
+        if sys.platform == 'darwin':
+            path += ':.'
+        unbuffer = []
+    env['PATH'] = path + os.pathsep + env['PATH']
+    env['DCS'] = what
+
+    ret = subprocess.call(unbuffer + [sys.executable, '-m', 'behave'], env=env)
+
+    if ret != 0:
+        if subprocess.call('grep . features/output/*_failed/*postgres?.*', shell=True) != 0:
+            subprocess.call('grep . features/output/*/*postgres?.*', shell=True)
+        return 1
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
@@ -0,0 +1,173 @@
+name: Tests
+
+on:
+  pull_request:
+  push:
+    branches:
+    - master
+    tags:
+    - v.*
+
+jobs:
+  unit:
+    runs-on: ${{ matrix.os }}-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu, windows, macos]
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 2.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 2.7
+      if: matrix.os != 'windows'
+    - name: Install dependencies
+      run: python .github/workflows/install_deps.py
+      if: matrix.os != 'windows'
+    - name: Run tests and flake8
+      run: python .github/workflows/run_tests.py
+      if: matrix.os != 'windows'
+
+    - name: Set up Python 3.5
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.5
+    - name: Install dependencies
+      run: python .github/workflows/install_deps.py
+    - name: Run tests and flake8
+      run: python .github/workflows/run_tests.py
+
+    - name: Set up Python 3.6
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.6
+    - name: Install dependencies
+      run: python .github/workflows/install_deps.py
+    - name: Run tests and flake8
+      run: python .github/workflows/run_tests.py
+
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+    - name: Install dependencies
+      run: python .github/workflows/install_deps.py
+    - name: Run tests and flake8
+      run: python .github/workflows/run_tests.py
+
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: python .github/workflows/install_deps.py
+    - name: Run tests and flake8
+      run: python .github/workflows/run_tests.py
+
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+    - name: Install dependencies
+      run: python .github/workflows/install_deps.py
+    - name: Run tests and flake8
+      run: python .github/workflows/run_tests.py
+
+    - name: Combine coverage
+      run: python .github/workflows/run_tests.py combine
+
+    - name: Install coveralls
+      run: python -m pip install coveralls
+
+    - name: Upload Coverage
+      env:
+        COVERALLS_FLAG_NAME: unit-${{ matrix.os }}
+        COVERALLS_PARALLEL: 'true'
+        GITHUB_TOKEN: ${{ secrets.github_token }}
+      run: python -m coveralls
+
+    - name: Run codacy-coverage-reporter
+      uses: codacy/codacy-coverage-reporter-action@master
+      with:
+        project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
+        coverage-reports: coverage.xml
+      if: matrix.os == 'ubuntu'
+
+  behave:
+    runs-on: ${{ matrix.os }}-latest
+    env:
+      DCS: ${{ matrix.dcs }}
+      ETCDVERSION: 3.3.13
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu]
+        python-version: [2.7, 3.5, 3.8]
+        dcs: [etcd, etcd3, consul, exhibitor, kubernetes, raft]
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: python .github/workflows/install_deps.py
+    - name: Run behave tests
+      run: python .github/workflows/run_tests.py
+    - uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+    - name: Install coveralls
+      run: python -m pip install coveralls
+    - name: Upload Coverage
+      env:
+        COVERALLS_FLAG_NAME: behave-${{ matrix.os }}-${{ matrix.dcs }}-${{ matrix.python-version }}
+        COVERALLS_PARALLEL: 'true'
+        GITHUB_TOKEN: ${{ secrets.github_token }}
+      run: python -m coveralls
+
+  behavem:
+    runs-on: ${{ matrix.os }}-latest
+    env:
+      DCS: ${{ matrix.dcs }}
+      ETCDVERSION: 3.3.13
+      PGVERSION: 12.1-1  # for windows and macos
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos]  #, windows]
+        python-version: [3.7]
+        dcs: [etcd, etcd3, raft]
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: python .github/workflows/install_deps.py
+    - name: Run behave tests
+      run: python .github/workflows/run_tests.py
+    - name: Install coveralls
+      run: python -m pip install coveralls
+    - name: Upload Coverage
+      env:
+        COVERALLS_FLAG_NAME: behave-${{ matrix.os }}-${{ matrix.dcs }}-${{ matrix.python-version }}
+        COVERALLS_PARALLEL: 'true'
+        GITHUB_TOKEN: ${{ secrets.github_token }}
+      run: python -m coveralls
+
+  coveralls-finish:
+    name: Finalize coveralls.io
+    needs: [unit, behave, behavem]
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/setup-python@v2
+    - run: python -m pip install coveralls
+    - run: python -m coveralls --finish
+      env:
+        GITHUB_TOKEN: ${{ secrets.github_token }}