From 03bb9125cbe7a353a970b27a1a4ac02bc8dc4777 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 7 May 2024 09:29:28 +0200 Subject: [PATCH] Compatibility with python 3.12 (#3058) - monkey patch `jsonlogger.RESERVED_ATTRS` to hide new attribute in `LogRecord` - "silence" warning about `atetime.datetime.utcnow()` - run some tests with python 3.12 - bump actions versions to silence complains about Node version - fix PATH to Postgres binaries on MacOS --- .github/workflows/install_deps.py | 11 +++---- .github/workflows/release.yaml | 12 ++++---- .github/workflows/run_tests.py | 2 +- .github/workflows/tests.yaml | 48 +++++++++++++++++++------------ patroni/log.py | 3 ++ tests/test_ha.py | 2 +- 6 files changed, 47 insertions(+), 31 deletions(-) diff --git a/.github/workflows/install_deps.py b/.github/workflows/install_deps.py index b089bde0..d976abbb 100644 --- a/.github/workflows/install_deps.py +++ b/.github/workflows/install_deps.py @@ -9,6 +9,11 @@ import zipfile def install_requirements(what): + subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'pip']) + s = subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'wheel', 'setuptools']) + if s != 0: + return s + old_path = sys.path[:] w = os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe()))) sys.path.insert(0, os.path.dirname(os.path.dirname(w))) @@ -28,11 +33,7 @@ def install_requirements(what): if not extras or what == 'all' or what in extras: requirements.append(r) - subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'pip']) - subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'wheel']) - r = subprocess.call([sys.executable, '-m', 'pip', 'install'] + requirements) - s = subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'setuptools']) - return s | r + return subprocess.call([sys.executable, '-m', 'pip', 'install'] + requirements) def install_packages(what): diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7a102379..d8657910 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -11,12 +11,12 @@ jobs: name: Build and publish Patroni distributions to PyPI and TestPyPI runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v4 - - name: Set up Python 3.9 - uses: actions/setup-python@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.11 - name: Install dependencies run: python .github/workflows/install_deps.py @@ -32,13 +32,13 @@ jobs: - name: Publish distribution to Test PyPI if: github.event_name == 'push' - uses: pypa/gh-action-pypi-publish@v1.5.1 + uses: pypa/gh-action-pypi-publish@v1.8.14 with: password: ${{ secrets.TEST_PYPI_API_TOKEN }} repository_url: https://test.pypi.org/legacy/ - name: Publish distribution to PyPI if: github.event_name == 'release' - uses: pypa/gh-action-pypi-publish@v1.5.1 + uses: pypa/gh-action-pypi-publish@v1.8.14 with: password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/run_tests.py b/.github/workflows/run_tests.py index cece186f..9a9d4db1 100644 --- a/.github/workflows/run_tests.py +++ b/.github/workflows/run_tests.py @@ -31,7 +31,7 @@ def main(): else: if sys.platform == 'darwin': version = os.environ.get('PGVERSION', '16.1-1') - path = '/usr/local/opt/postgresql@{0}/bin:.'.format(version.split('.')[0]) + path = '/opt/homebrew/opt/postgresql@{0}/bin:.'.format(version.split('.')[0]) unbuffer = ['unbuffer'] else: path = os.path.abspath(os.path.join('pgsql', 'bin')) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 311aed66..37b94f9c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -20,19 +20,22 @@ jobs: os: [ubuntu, windows, macos] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.7 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.7 + if: matrix.os != 'macos' - name: Install dependencies run: python .github/workflows/install_deps.py + if: matrix.os != 'macos' - name: Run tests and flake8 run: python .github/workflows/run_tests.py + if: matrix.os != 'macos' - name: Set up Python 3.8 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.8 - name: Install dependencies @@ -41,7 +44,7 @@ jobs: run: python .github/workflows/run_tests.py - name: Set up Python 3.9 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.9 - name: Install dependencies @@ -50,7 +53,7 @@ jobs: run: python .github/workflows/run_tests.py - name: Set up Python 3.10 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.10' - name: Install dependencies @@ -59,7 +62,7 @@ jobs: run: python .github/workflows/run_tests.py - name: Set up Python 3.11 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.11 - name: Install dependencies @@ -67,6 +70,15 @@ jobs: - name: Run tests and flake8 run: python .github/workflows/run_tests.py + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: 3.12 + - name: Install dependencies + run: python .github/workflows/install_deps.py + - name: Run tests and flake8 + run: python .github/workflows/run_tests.py + - name: Combine coverage run: python .github/workflows/run_tests.py combine @@ -90,7 +102,7 @@ jobs: fail-fast: false matrix: os: [ubuntu] - python-version: [3.7, '3.10'] + python-version: [3.7, 3.12] dcs: [etcd, etcd3, consul, exhibitor, kubernetes, raft] include: - os: macos @@ -104,9 +116,9 @@ jobs: dcs: etcd3 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - uses: nolar/setup-k3d-k3s@v1 @@ -125,7 +137,7 @@ jobs: - name: Run behave tests run: python .github/workflows/run_tests.py - name: Upload logs if behave failed - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: name: behave-${{ matrix.os }}-${{ matrix.dcs }}-${{ matrix.python-version }}-logs @@ -145,7 +157,7 @@ jobs: needs: unit runs-on: ubuntu-latest steps: - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 - run: python -m pip install coveralls - run: python -m coveralls --service=github --finish env: @@ -162,27 +174,27 @@ jobs: pyright: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - name: Set up Python 3.11 - uses: actions/setup-python@v4 + - name: Set up Python 3.12 + uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: 3.12 - name: Install dependencies run: python -m pip install -r requirements.txt psycopg2-binary psycopg - - uses: jakebailey/pyright-action@v1 + - uses: jakebailey/pyright-action@v2 with: version: 1.1.356 docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.11 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.11 cache: pip diff --git a/patroni/log.py b/patroni/log.py index 17752056..3a7b55e7 100644 --- a/patroni/log.py +++ b/patroni/log.py @@ -350,6 +350,9 @@ class PatroniLogger(Thread): try: from pythonjsonlogger import jsonlogger + if hasattr(jsonlogger, 'RESERVED_ATTRS') and 'taskName' not in jsonlogger.RESERVED_ATTRS: + # compatibility with python 3.12, that added a new attribute to LogRecord + jsonlogger.RESERVED_ATTRS += ('taskName',) return jsonlogger.JsonFormatter( jsonformat, diff --git a/tests/test_ha.py b/tests/test_ha.py index abfb4691..d6f70124 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -790,7 +790,7 @@ class TestHa(PostgresInit): self.assertIn('Incorrect value of scheduled_at: %s', mock_warning.call_args_list[0][0]) # scheduled now - scheduled = datetime.datetime.utcnow().replace(tzinfo=tzutc) + scheduled = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=tzutc) self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'b', scheduled)) self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'})) self.assertEqual('switchover: demoting myself', self.ha.run_cycle())