From 73797e85728cc2ba431d6a309515b7e95a4966f5 Mon Sep 17 00:00:00 2001 From: Matt Baker <93600443+matthbakeredb@users.noreply.github.com> Date: Wed, 24 May 2023 09:58:04 +0100 Subject: [PATCH] Add tox configuration for running multiple test envs (#2603) --- docs/CONTRIBUTING.rst | 122 +++++++++++++++++++++++++- features/Dockerfile | 91 ++++++++++++++++++++ pyrightconfig.json | 2 +- tox.ini | 195 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 406 insertions(+), 4 deletions(-) create mode 100644 features/Dockerfile diff --git a/docs/CONTRIBUTING.rst b/docs/CONTRIBUTING.rst index 58e5f7bc..44733a55 100644 --- a/docs/CONTRIBUTING.rst +++ b/docs/CONTRIBUTING.rst @@ -19,7 +19,7 @@ Requirements for running behave tests: 2. PostgreSQL binaries must be available in your `PATH`. You may need to add them to the path with something like `PATH=/usr/lib/postgresql/11/bin:$PATH python -m behave`. 3. If you'd like to test with external DCSs (e.g., Etcd, Consul, and Zookeeper) you'll need the packages installed and respective services running and accepting unencrypted/unprotected connections on localhost and default port. In the case of Etcd or Consul, the behave test suite could start them up if binaries are available in the `PATH`. - Install dependencies: +Install dependencies: .. code-block:: bash @@ -43,6 +43,126 @@ After you have all dependencies installed, you can run the various test suites: # modify DCS as desired (raft has no dependencies so is the easiest to start with): DCS=raft python -m behave +Testing with tox +---------------- + +To run tox tests you only need to install one dependency (other than Python) + +.. code-block:: bash + + pip install tox>=4 + +If you wish to run `behave` tests then you also need docker installed. + +Tox configuration in `tox.ini` has "environments" to run the following tasks: + +* lint: Python code lint with `flake8` +* test: unit tests for all available python interpreters with `pytest`, + generates XML reports or HTML reports if a TTY is detected +* dep: detect package dependency conflicts using `pipdeptree` +* type: static type checking with `pyright` +* black: code formatting with `black` +* docker-build: build docker image used for the `behave` env +* docker-cmd: run arbitrary command with the above image +* docker-behave-etcd: run tox for behave tests with above image +* py*behave: run behave with available python interpreters (without docker, although + this is what is called inside docker containers) +* docs: build docs with `sphinx` + +Running tox +^^^^^^^^^^^ + +To run the default env list; dep, lint, test, and docs, just run: + +.. code-block:: bash + + tox + +The `test` envs can be run with the label `test`: + +.. code-block:: bash + + tox -m test + +The `behave` docker tests can be run with the label `behave`: + +.. code-block:: bash + + tox -m behave + +Similarly, docs has the label `docs`. + +All other envs can be run with their respective env names: + +.. code-block:: bash + + tox -e lint + tox -e py39-test-lin + +It is also possible to select partial env lists using `factors`. For example, if you want to run +all envs for python 3.10: + +.. code-block:: bash + + tox -f py310 + +This is equivalent to running all the envs listed below: + +.. code-block:: bash + + $ tox -l -f py310 + py310-test-lin + py310-test-mac + py310-test-win + py310-type-lin + py310-type-mac + py310-type-win + py310-behave-etcd-lin + py310-behave-etcd-win + py310-behave-etcd-mac + + +You can list all configured combinations of environments with tox (>=v4) like so + +.. code-block:: bash + + tox l + +The envs `test` and `docs` will attempt to open the HTML output files +when the job completes, if tox is run with an active terminal. This +is intended to be for benefit of the developer running this env locally. +It will attempt to run `open` on a mac and `xdg-open` on Linux. +To use a different command set the env var `OPEN_CMD` to the name or path of +the command. If this step fails it will not fail the run overall. +If you want to disable this facility set the env var `OPEN_CMD` to the `:` no-op command. + +.. code-block:: bash + + OPEN_CMD=: tox -m docs + +Behave tests +^^^^^^^^^^^^ + +Behave tests with `-m behave` will build docker images based on PG_MAJOR version 11 through 15 and then run all +behave tests. This can take quite a long time to run so you might want to limit the scope to a select version of +Postgres or to a specific feature set or steps. + +To specify the version of postgres include the full name of the dependent image build env that you want and then the +behave env name. For instance if you want Postgres 15 use: + +.. code-block:: bash + + tox -e pg14-docker-build,pg14-docker-behave-etcd-lin + +If on the other hand you want to test a specific feature you can pass positional arguments to behave. This will run +the watchdog behave feature test scenario with all versions of Postgres. + +.. code-block:: bash + + tox -m behave -- features/watchdog.feature + +Of course you can combine the two. + Reporting issues ---------------- diff --git a/features/Dockerfile b/features/Dockerfile new file mode 100644 index 00000000..e2f5dbb9 --- /dev/null +++ b/features/Dockerfile @@ -0,0 +1,91 @@ +# syntax = docker/dockerfile:1.5 +# Used only for running tests using tox, see ../tox.ini +ARG PG_MAJOR +ARG PGHOME=/home/postgres +ARG LC_ALL=C.UTF-8 +ARG LANG=C.UTF-8 + +FROM postgres:${PG_MAJOR} + +ARG PGHOME +ARG LC_ALL +ARG LANG + +ENV PGHOME="$PGHOME" +ENV LC_ALL="$LC_ALL" +ENV LANG="$LANG" + +ARG ETCDVERSION=3.3.13 +ENV ETCDVERSION="$ETCDVERSION" +ARG ETCDURL="https://github.com/coreos/etcd/releases/download/v$ETCDVERSION" + +USER root +RUN set -ex \ + && apt-get update \ + && apt-get reinstall init-system-helpers \ + && apt-get install -y \ + python3-pip \ + python3-dev \ + rsync \ + curl \ + gcc \ + golang \ + jq \ + locales \ + sudo \ + busybox \ + net-tools \ + iputils-ping \ + && rm -rf /var/cache/apt \ + && python3 -m pip install --no-cache-dir tox \ + \ + && mkdir -p "$PGHOME" \ + && sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \ + && chown -R postgres:postgres /var/log /home/postgres \ + \ + # Download etcd \ + && curl -sL "$ETCDURL/etcd-v$ETCDVERSION-linux-$(dpkg --print-architecture).tar.gz" \ + | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl + + +# This Dockerfile syntax only works with docker buildx and the syntax +# line at the top of this file. +COPY </dev/null \\ + | sed 's|^./||' >/tmp/copy_exclude.lst \\ + || true +runuser -u postgres -- \\ + rsync -a \\ + --exclude=.tox \\ + --exclude="features/output*" \\ + --exclude-from="/tmp/copy_exclude.lst" \\ + . "\$PGHOME/src/" +cd "\$PGHOME/src" +runuser -u postgres -w ETCD_UNSUPPORTED_ARCH -- "\$@" & +wait $! +# SIGINT whilst child proc is running is not seen by trap so we run a copy here instead of using +# trap copy_output SIGINT EXIT +copy_output +EOF +RUN chmod +x /tox-wrapper.sh + +VOLUME /src + +ENTRYPOINT ["/tox-wrapper.sh"] diff --git a/pyrightconfig.json b/pyrightconfig.json index 2394c6de..91bed713 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -2,7 +2,7 @@ "include": [ "patroni" ], - + "exclude": [ "**/__pycache__" ], diff --git a/tox.ini b/tox.ini index 67616ade..72353d4f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,3 +1,194 @@ +[common] +python_matrix = {36,37,38,39,310,311} +postgres_matrix = + pg11: PG_MAJOR = 11 + pg12: PG_MAJOR = 12 + pg13: PG_MAJOR = 13 + pg14: PG_MAJOR = 14 + pg15: PG_MAJOR = 15 +psycopg_deps = + py{37,38,39,310,311}-{lin,win}: psycopg[binary] + mac: psycopg2-binary + py36: psycopg2-binary +platforms = + lin: linux + mac: darwin + win: win32 + +[tox] +min_version = 4.0 +requires = + tox>4 +env_list = + dep + lint + py{[common]python_matrix}-test-{lin,mac,win} + docs +skipsdist = True +toxworkdir = {env:TOX_WORK_DIR:.tox} +skip_missing_interpreters = True + +[testenv] +setenv = + PYTHONDONTWRITEBYTECODE = 1 + mac: OPEN_CMD = {env:OPEN_CMD:open} + lin: OPEN_CMD = {env:OPEN_CMD:xdg-open} +passenv = + BROWSER + DISPLAY + +[testenv:lint] +description = Lint code with flake8 +commands = flake8 {posargs:patroni tests setup.py} +deps = + flake8 + +[testenv:py{36,37,38,39,310,311}-test-{lin,win,mac}] +description = Run unit tests with pytest +labels = + test +commands_pre = + - {tty:rm -f "{toxworkdir}{/}cov_report_{env_name}_html{/}index.html":true} + - {tty:rm -f "{toxworkdir}{/}pytest_report_{env_name}.html":true} +commands = + pytest \ + -p no:cacheprovider \ + --verbose \ + --doctest-modules \ + --capture=fd \ + --cov=patroni \ + --cov-report=term-missing \ + --cov-append \ + {tty::--cov-report="xml\:{toxworkdir}{/}cov_report.{env_name}.xml"} \ + {tty:--cov-report="html\:{toxworkdir}{/}cov_report_{env_name}_html":} \ + {tty:--html="{toxworkdir}{/}pytest_report_{env_name}.html":} \ + {posargs:tests patroni} +commands_post = + - {tty:{env:OPEN_CMD} "{toxworkdir}{/}cov_report_{env_name}_html{/}index.html":true} + - {tty:{env:OPEN_CMD} "{toxworkdir}{/}pytest_report_{env_name}.html":true} +deps = + -r requirements.txt + mock>=2.0.0 + pytest + pytest-cov + pytest-html + {[common]psycopg_deps} +platform = + {[common]platforms} +allowlist_externals = + rm + {env:OPEN_CMD} + +[testenv:dep] +description = Check package dependency problems +commands = pipdeptree -w fail +deps = + -r requirements.txt + pipdeptree + {[common]psycopg_deps} + +[testenv:py{37,38,39,310,311}-type-{lin,mac,win}] +description = Run static type checking with pyright +labels = + type +deps = + -r requirements.txt + pyright + psycopg2-binary + psycopg[binary] +commands = pyright --venv-path {toxworkdir}{/}{envname} {posargs:patroni} +platform = + {[common]platforms} + +[testenv:black] +description = Reformat code with black +deps = black +commands = black {posargs:patroni tests} + +[testenv:pg{12,13,14,15}-docker-build] +description = Build docker containers needed for testing +labels = + behave + docker-build +setenv = + {[common]postgres_matrix} + DOCKER_BUILDKIT = 1 +commands = + docker build . \ + --tag patroni-dev:{env:PG_MAJOR} \ + --build-arg PG_MAJOR \ + --file features/Dockerfile +allowlist_externals = docker + +[testenv:pg{12,13,14,15}-docker-behave-{etcd}-{lin,mac}] +description = Run behaviour tests in patroni-dev docker container +setenv = + etcd: DCS=etcd + {[common]postgres_matrix} + CONTAINER_NAME = tox-{env_name}-{env:PYTHONHASHSEED} +labels = + behave +depends = + pg{11,12,13,14,15}-docker-build + +# There's a bug which affects calling multiple envs on the command line +# This should be a valid command: tox -e 'py{36,37,38,39,310,311}-behave-{env:DCS}-lin' +# Replaced with workaround, see https://github.com/tox-dev/tox/issues/2850 +commands = + docker run \ + --volume {tox_root}:/src \ + --env DCS={env:DCS} \ + --hostname {env:CONTAINER_NAME} \ + --name {env:CONTAINER_NAME} \ + --rm \ + --tty \ + {env:PATRONI_DEV_IMAGE:patroni-dev:{env:PG_MAJOR}} \ + tox run -x 'tox.env_list=py{[common]python_matrix}-behave-{env:DCS}-lin' \ + -- --format plain {posargs} + +allowlist_externals = + docker + find +platform = + lin: linux +; win: win32 + mac: darwin + +[testenv:py{36,38,39,310,311}-behave-{etcd}-{lin,win,mac}] +description = Run behaviour tests (locally with tox) +deps = + -r requirements.txt + behave + coverage + {[common]psycopg_deps} +setenv = + DCS = {env:DCS:etcd} +passenv = + ETCD_UNSUPPORTED_ARCH +commands = + python3 -m behave {posargs} +platform = + {[common]platforms} + +[testenv:docs-{lin,mac,win}] +description = Build Sphinx documentation +labels: + docs +deps = + sphinx>=4 + sphinx_rtd_theme +commands = + sphinx-build \ + -d "{envtmpdir}{/}doctree" docs "{toxworkdir}{/}docs_out" \ + --color -b html \ + {posargs} +commands_post = + - {tty:{env:OPEN_CMD} "{toxworkdir}{/}docs_out{/}index.html":true:} +allowlist_externals = + {env:OPEN_CMD} +platform = + {[common]platforms} + [flake8] -max-line-length=120 -ignore=D401,W503 +max-line-length = 120 +ignore = D401,W503