Compatibility with python 3.12 (#3058)

- monkey patch `jsonlogger.RESERVED_ATTRS` to hide new attribute in `LogRecord`
- "silence" warning about `atetime.datetime.utcnow()`
- run some tests with python 3.12
- bump actions versions to silence complains about Node version
- fix PATH to Postgres binaries on MacOS

.github/workflows/install_deps.py

@@ -9,6 +9,11 @@ import zipfile
 
 
 def install_requirements(what):
+    subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'pip'])
+    s = subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'wheel', 'setuptools'])
+    if s != 0:
+        return s
+
     old_path = sys.path[:]
     w = os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe())))
     sys.path.insert(0, os.path.dirname(os.path.dirname(w)))
@@ -28,11 +33,7 @@ def install_requirements(what):
             if not extras or what == 'all' or what in extras:
                 requirements.append(r)
 
-    subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'pip'])
+    return subprocess.call([sys.executable, '-m', 'pip', 'install'] + requirements)
-    subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'wheel'])
-    r = subprocess.call([sys.executable, '-m', 'pip', 'install'] + requirements)
-    s = subprocess.call([sys.executable, '-m', 'pip', 'install', '--upgrade', 'setuptools'])
-    return s | r
 
 
 def install_packages(what):

.github/workflows/release.yaml

@@ -11,12 +11,12 @@ jobs:
     name: Build and publish Patroni distributions to PyPI and TestPyPI
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v4
 
-    - name: Set up Python 3.9
+    - name: Set up Python 3.11
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.9
+        python-version: 3.11
 
     - name: Install dependencies
       run: python .github/workflows/install_deps.py
@@ -32,13 +32,13 @@ jobs:
 
     - name: Publish distribution to Test PyPI
       if: github.event_name == 'push'
-      uses: pypa/gh-action-pypi-publish@v1.5.1
+      uses: pypa/gh-action-pypi-publish@v1.8.14
       with:
         password: ${{ secrets.TEST_PYPI_API_TOKEN }}
         repository_url: https://test.pypi.org/legacy/
 
     - name: Publish distribution to PyPI
       if: github.event_name == 'release'
-      uses: pypa/gh-action-pypi-publish@v1.5.1
+      uses: pypa/gh-action-pypi-publish@v1.8.14
       with:
         password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/run_tests.py

@@ -31,7 +31,7 @@ def main():
     else:
         if sys.platform == 'darwin':
             version = os.environ.get('PGVERSION', '16.1-1')
-            path = '/usr/local/opt/postgresql@{0}/bin:.'.format(version.split('.')[0])
+            path = '/opt/homebrew/opt/postgresql@{0}/bin:.'.format(version.split('.')[0])
             unbuffer = ['unbuffer']
         else:
             path = os.path.abspath(os.path.join('pgsql', 'bin'))

.github/workflows/tests.yaml

@@ -20,19 +20,22 @@ jobs:
         os: [ubuntu, windows, macos]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python 3.7
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.7
+      if: matrix.os != 'macos'
     - name: Install dependencies
       run: python .github/workflows/install_deps.py
+      if: matrix.os != 'macos'
     - name: Run tests and flake8
       run: python .github/workflows/run_tests.py
+      if: matrix.os != 'macos'
 
     - name: Set up Python 3.8
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.8
     - name: Install dependencies
@@ -41,7 +44,7 @@ jobs:
       run: python .github/workflows/run_tests.py
 
     - name: Set up Python 3.9
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.9
     - name: Install dependencies
@@ -50,7 +53,7 @@ jobs:
       run: python .github/workflows/run_tests.py
 
     - name: Set up Python 3.10
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.10'
     - name: Install dependencies
@@ -59,7 +62,7 @@ jobs:
       run: python .github/workflows/run_tests.py
 
     - name: Set up Python 3.11
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.11
     - name: Install dependencies
@@ -67,6 +70,15 @@ jobs:
     - name: Run tests and flake8
       run: python .github/workflows/run_tests.py
 
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.12
+    - name: Install dependencies
+      run: python .github/workflows/install_deps.py
+    - name: Run tests and flake8
+      run: python .github/workflows/run_tests.py
+
     - name: Combine coverage
       run: python .github/workflows/run_tests.py combine
 
@@ -90,7 +102,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu]
-        python-version: [3.7, '3.10']
+        python-version: [3.7, 3.12]
         dcs: [etcd, etcd3, consul, exhibitor, kubernetes, raft]
         include:
         - os: macos
@@ -104,9 +116,9 @@ jobs:
           dcs: etcd3
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - uses: nolar/setup-k3d-k3s@v1
@@ -125,7 +137,7 @@ jobs:
     - name: Run behave tests
       run: python .github/workflows/run_tests.py
     - name: Upload logs if behave failed
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       if: failure()
       with:
         name: behave-${{ matrix.os }}-${{ matrix.dcs }}-${{ matrix.python-version }}-logs
@@ -145,7 +157,7 @@ jobs:
     needs: unit
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
     - run: python -m pip install coveralls
     - run: python -m coveralls --service=github --finish
       env:
@@ -162,27 +174,27 @@ jobs:
   pyright:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
-    - name: Set up Python 3.11
+    - name: Set up Python 3.12
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.11
+        python-version: 3.12
 
     - name: Install dependencies
       run: python -m pip install -r requirements.txt psycopg2-binary psycopg
 
-    - uses: jakebailey/pyright-action@v1
+    - uses: jakebailey/pyright-action@v2
       with:
         version: 1.1.356
 
   docs:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python 3.11
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.11
         cache: pip

patroni/log.py

@@ -350,6 +350,9 @@ class PatroniLogger(Thread):
 
         try:
             from pythonjsonlogger import jsonlogger
+            if hasattr(jsonlogger, 'RESERVED_ATTRS') and 'taskName' not in jsonlogger.RESERVED_ATTRS:
+                # compatibility with python 3.12, that added a new attribute to LogRecord
+                jsonlogger.RESERVED_ATTRS += ('taskName',)
 
             return jsonlogger.JsonFormatter(
                 jsonformat,

tests/test_ha.py

@@ -790,7 +790,7 @@ class TestHa(PostgresInit):
             self.assertIn('Incorrect value of scheduled_at: %s', mock_warning.call_args_list[0][0])
 
         # scheduled now
-        scheduled = datetime.datetime.utcnow().replace(tzinfo=tzutc)
+        scheduled = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=tzutc)
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'b', scheduled))
         self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
         self.assertEqual('switchover: demoting myself', self.ha.run_cycle())