Allow to specify psycopg* in extras and switch to build (#2907)

* remove check_psycopg() call from the setup.py, when installing from wheel it doesn't work anyway.
* call check_psycopg() function before process_arguments(), because the last one is trying to import psycopg and fails with the stacktrace, while the first one shows a nice human-readable error message.
* add psycopg2, psycopg2-binary, and psycopg3 extras, that will install psycopg2>=2.5.4, psycopg2-binary, or psycopg[binary]>=3.0.0 modules respectively.
* move check_psycopg() function to the __main__.py.
* introduce the new extra called `all`, it will allow to install all dependencies at once (except psycopg related).
* use the `build` module in order to create sdist bdist_wheel packages.
* update the documentation regarding psycopg and extras (dependencies).

.github/workflows/release.yaml

@@ -24,8 +24,11 @@ jobs:
     - name: Run tests and flake8
       run: python .github/workflows/run_tests.py
 
+    - name: Install Python packaging build frontend
+      run: python -m pip install build
+
     - name: Build a binary wheel and a source tarball
-      run: python setup.py sdist bdist_wheel
+      run: python -m build
 
     - name: Publish distribution to Test PyPI
       if: github.event_name == 'push'

README.rst

@@ -77,23 +77,8 @@ There are a few options available:
     sudo apt-get install python3-psycopg2  # install psycopg2 module on Debian/Ubuntu
     sudo yum install python3-psycopg2      # install psycopg2 on RedHat/Fedora/CentOS
 
-2. Install psycopg2 from the binary package
+2. Specify one of `psycopg`, `psycopg2`, or `psycopg2-binary` in the list of dependencies when installing Patroni with pip (see below).
 
-::
-
-    pip install psycopg2-binary
-
-3. Install psycopg2 from source
-
-::
-
-    pip install psycopg2>=2.5.4
-
-4. Use psycopg 3.0 instead of psycopg2
-
-::
-
-    pip install psycopg[binary]>=3.0.0
 
 **General installation for pip**
 
@@ -119,12 +104,20 @@ raft
     `pysyncobj` module in order to use python Raft implementation as DCS
 aws
     `boto3` in order to use AWS callbacks
+all
+    all of the above (except psycopg family)
+psycopg3
+    `psycopg[binary]>=3.0.0` module
+psycopg2
+    `psycopg2>=2.5.4` module
+psycopg2-binary
+    `psycopg2-binary` module
 
-For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is:
+For example, the command in order to install Patroni together with psycopg3, dependencies for Etcd as a DCS, and AWS callbacks is:
 
 ::
 
-    pip install patroni[etcd,aws]
+    pip install patroni[psycopg3,etcd3,aws]
 
 Note that external tools to call in the replica creation or custom bootstrap scripts (i.e. WAL-E) should be installed independently of Patroni.
 

docs/installation.rst

@@ -30,23 +30,10 @@ There are a few options available:
     sudo apt-get install python3-psycopg2  # install psycopg2 module on Debian/Ubuntu
     sudo yum install python3-psycopg2      # install psycopg2 on RedHat/Fedora/CentOS
 
-2. Install psycopg2 from the binary package
+2. Specify one of `psycopg`, `psycopg2`, or `psycopg2-binary` in the :ref:`list of dependencies <extras>` when installing Patroni with pip.
 
-.. code-block:: shell
 
-    pip install psycopg2-binary
-
-3. Install psycopg2 from source
-
-.. code-block:: shell
-
-    pip install psycopg2>=2.5.4
-
-4. Use psycopg 3.0 instead of psycopg2
-
-.. code-block:: shell
-
-    pip install psycopg[binary]>=3.0.0
+.. _extras:
 
 General installation for pip
 ----------------------------
@@ -73,12 +60,20 @@ raft
     `pysyncobj` module in order to use python Raft implementation as DCS
 aws
     `boto3` in order to use AWS callbacks
+all
+    all of the above (except psycopg family)
+psycopg
+    `psycopg[binary]>=3.0.0` module
+psycopg2
+    `psycopg2>=2.5.4` module
+psycopg2-binary
+    `psycopg2-binary` module
 
-For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is:
+For example, the command in order to install Patroni together with psycopg3, dependencies for Etcd as a DCS, and AWS callbacks is:
 
 .. code-block:: shell
 
-    pip install patroni[etcd,aws]
+    pip install patroni[psycopg3,etcd3,aws]
 
 Note that external tools to call in the replica creation or custom bootstrap scripts (i.e. WAL-E) should be installed
 independently of Patroni.

patroni/__init__.py

@@ -3,23 +3,14 @@
 :var PATRONI_ENV_PREFIX: prefix for Patroni related configuration environment variables.
 :var KUBERNETES_ENV_PREFIX: prefix for Kubernetes related configuration environment variables.
 :var MIN_PSYCOPG2: minimum version of :mod:`psycopg2` required by Patroni to work.
+:var MIN_PSYCOPG3: minimum version of :mod:`psycopg` required by Patroni to work.
 """
-
-import sys
-
-from typing import Any, Callable, Iterator, Tuple
+from typing import Iterator, Tuple
 
 PATRONI_ENV_PREFIX = 'PATRONI_'
 KUBERNETES_ENV_PREFIX = 'KUBERNETES_'
 MIN_PSYCOPG2 = (2, 5, 4)
-
-
-def fatal(string: str, *args: Any) -> None:
-    """Write a fatal message to stderr and exit with code ``1``.
-
-    :param string: message to be written before exiting.
-    """
-    sys.exit('FATAL: ' + string.format(*args))
+MIN_PSYCOPG3 = (3, 0, 0)
 
 
 def parse_version(version: str) -> Tuple[int, ...]:
@@ -28,25 +19,25 @@ def parse_version(version: str) -> Tuple[int, ...]:
     .. note::
         Designed for easy comparison of software versions in Python.
 
-    :param version: human-readable software version, e.g. ``2.5.4``.
+    :param version: human-readable software version, e.g. ``2.5.4.dev1 (dt dec pq3 ext lo64)``.
 
     :returns: tuple of *version* parts, each part as an integer.
 
     :Example:
 
-        >>> parse_version('2.5.4')
+        >>> parse_version('2.5.4.dev1 (dt dec pq3 ext lo64)')
         (2, 5, 4)
     """
     def _parse_version(version: str) -> Iterator[int]:
         """Yield each part of a human-readable version string as an integer.
 
-        :param version: human-readable software version, e.g. ``2.5.4``.
+        :param version: human-readable software version, e.g. ``2.5.4.dev1``.
 
         :yields: each part of *version* as an integer.
 
         :Example:
 
-            >>> tuple(_parse_version('2.5.4'))
+            >>> tuple(_parse_version('2.5.4.dev1'))
             (2, 5, 4)
         """
         for e in version.split('.'):
@@ -55,40 +46,3 @@ def parse_version(version: str) -> Tuple[int, ...]:
             except ValueError:
                 break
     return tuple(_parse_version(version.split(' ')[0]))
-
-
-def check_psycopg(_min_psycopg2: Tuple[int, ...] = MIN_PSYCOPG2,
-                  _parse_version: Callable[[str], Tuple[int, ...]] = parse_version) -> None:
-    """Ensure at least one among :mod:`psycopg2` or :mod:`psycopg` libraries are available in the environment.
-
-    .. note::
-        We pass ``MIN_PSYCOPG2`` and :func:`parse_version` as arguments to simplify usage of :func:`check_psycopg` from
-        the ``setup.py``.
-
-    .. note::
-        Patroni chooses :mod:`psycopg2` over :mod:`psycopg`, if possible.
-
-        If nothing meeting the requirements is found, then exit with a fatal message.
-
-    :param _min_psycopg2: minimum required version in case :mod:`psycopg2` is chosen.
-    :param _parse_version: function used to parse :mod:`psycopg2`/:mod:`psycopg` version into a comparable object.
-    """
-    min_psycopg2_str = '.'.join(map(str, _min_psycopg2))
-
-    # try psycopg2
-    try:
-        from psycopg2 import __version__
-        if _parse_version(__version__) >= _min_psycopg2:
-            return
-        version_str = __version__.split(' ')[0]
-    except ImportError:
-        version_str = None
-
-    # try psycopg3
-    try:
-        from psycopg import __version__
-    except ImportError:
-        error = 'Patroni requires psycopg2>={0}, psycopg2-binary, or psycopg>=3.0'.format(min_psycopg2_str)
-        if version_str is not None:
-            error += ', but only psycopg2=={0} is available'.format(version_str)
-        fatal(error)

patroni/__main__.py

@@ -10,8 +10,9 @@ import sys
 import time
 
 from argparse import Namespace
-from typing import Any, Dict, Optional, TYPE_CHECKING
+from typing import Any, Dict, List, Optional, TYPE_CHECKING
 
+from patroni import MIN_PSYCOPG2, MIN_PSYCOPG3, parse_version
 from patroni.daemon import AbstractPatroniDaemon, abstract_main, get_base_arg_parser
 from patroni.tags import Tags
 
@@ -286,6 +287,45 @@ def process_arguments() -> Namespace:
     return args
 
 
+def check_psycopg() -> None:
+    """Ensure at least one among :mod:`psycopg2` or :mod:`psycopg` libraries are available in the environment.
+
+    .. note::
+        Patroni chooses :mod:`psycopg2` over :mod:`psycopg`, if possible.
+
+        If nothing meeting the requirements is found, then exit with a fatal message.
+    """
+    min_psycopg2_str = '.'.join(map(str, MIN_PSYCOPG2))
+    min_psycopg3_str = '.'.join(map(str, MIN_PSYCOPG3))
+
+    available_versions: List[str] = []
+
+    # try psycopg2
+    try:
+        from psycopg2 import __version__
+        if parse_version(__version__) >= MIN_PSYCOPG2:
+            return
+        available_versions.append('psycopg2=={0}'.format(__version__.split(' ')[0]))
+    except ImportError:
+        logger.debug('psycopg2 module is not available')
+
+    # try psycopg3
+    try:
+        from psycopg import __version__
+        if parse_version(__version__) >= MIN_PSYCOPG3:
+            return
+        available_versions.append('psycopg=={0}'.format(__version__.split(' ')[0]))
+    except ImportError:
+        logger.debug('psycopg module is not available')
+
+    error = f'FATAL: Patroni requires psycopg2>={min_psycopg2_str}, psycopg2-binary, or psycopg>={min_psycopg3_str}'
+    if available_versions:
+        error += ', but only {0} {1} available'.format(
+            ' and '.join(available_versions),
+            'is' if len(available_versions) == 1 else 'are')
+    sys.exit(error)
+
+
 def main() -> None:
     """Main entrypoint of :mod:`patroni.__main__`.
 
@@ -297,12 +337,10 @@ def main() -> None:
         ``patroni`` daemon as another process. In that case relevant signals received by the main process and forwarded
         to ``patroni`` daemon process.
     """
-    from patroni import check_psycopg
+    check_psycopg()
 
     args = process_arguments()
 
-    check_psycopg()
-
     if os.getpid() != 1:
         return patroni_main(args.configfile)
 

setup.py

@@ -26,7 +26,6 @@ KEYWORDS = 'etcd governor patroni postgresql postgres ha haproxy confd' +\
 EXTRAS_REQUIRE = {'aws': ['boto3'], 'etcd': ['python-etcd'], 'etcd3': ['python-etcd'],
                   'consul': ['python-consul'], 'exhibitor': ['kazoo'], 'zookeeper': ['kazoo'],
                   'kubernetes': [], 'raft': ['pysyncobj', 'cryptography']}
-COVERAGE_XML = True
 
 # Add here all kinds of additional classifiers as defined under
 # https://pypi.python.org/pypi?%3Aaction=list_classifiers
@@ -120,14 +119,21 @@ def read(fname):
         return fd.read()
 
 
-def setup_package(version):
+def get_versions():
+    old_modules = sys.modules.copy()
+    try:
+        from patroni import MIN_PSYCOPG2, MIN_PSYCOPG3
+        from patroni.version import __version__
+        return __version__, MIN_PSYCOPG2, MIN_PSYCOPG3
+    finally:
+        sys.modules.clear()
+        sys.modules.update(old_modules)
+
+
+def main():
     logging.basicConfig(format='%(message)s', level=os.getenv('LOGLEVEL', logging.WARNING))
 
-    # Assemble additional setup commands
-    cmdclass = {'test': PyTest, 'flake8': Flake8}
-
     install_requires = []
-
     for r in read('requirements.txt').split('\n'):
         r = r.strip()
         if r == '':
@@ -139,15 +145,22 @@ def setup_package(version):
                     deps[i] = r
                     EXTRAS_REQUIRE[e] = deps
                     extra = True
-                    break
-            if extra:
-                break
         if not extra:
             install_requires.append(r)
 
+    # Just for convenience, if someone wants to install dependencies for all extras
+    EXTRAS_REQUIRE['all'] = list({e for extras in EXTRAS_REQUIRE.values() for e in extras})
+
+    patroni_version, min_psycopg2, min_psycopg3 = get_versions()
+
+    # Make it possible to specify psycopg dependency as extra
+    for name, version in {'psycopg[binary]': min_psycopg3, 'psycopg2': min_psycopg2, 'psycopg2-binary': None}.items():
+        EXTRAS_REQUIRE[name] = [name + ('>=' + '.'.join(map(str, version)) if version else '')]
+    EXTRAS_REQUIRE['psycopg3'] = EXTRAS_REQUIRE.pop('psycopg[binary]')
+
     setup(
         name=NAME,
-        version=version,
+        version=patroni_version,
         url=URL,
         author=AUTHOR,
         author_email=AUTHOR_EMAIL,
@@ -163,20 +176,10 @@ def setup_package(version):
         ]},
         install_requires=install_requires,
         extras_require=EXTRAS_REQUIRE,
-        cmdclass=cmdclass,
+        cmdclass={'test': PyTest, 'flake8': Flake8},
         entry_points={'console_scripts': CONSOLE_SCRIPTS},
     )
 
 
 if __name__ == '__main__':
-    old_modules = sys.modules.copy()
-    try:
-        from patroni import check_psycopg
-        from patroni.version import __version__
-    finally:
-        sys.modules.clear()
-        sys.modules.update(old_modules)
-
-    check_psycopg()
-
-    setup_package(__version__)
+    main()

tests/test_patroni.py

@@ -15,8 +15,7 @@ from patroni.dcs.etcd import AbstractEtcdClientWithFailover
 from patroni.exceptions import DCSError
 from patroni.postgresql import Postgresql
 from patroni.postgresql.config import ConfigHandler
-from patroni import check_psycopg
-from patroni.__main__ import Patroni, main as _main
+from patroni.__main__ import check_psycopg, Patroni, main as _main
 from threading import Thread
 
 from . import psycopg_connect, SleepException
@@ -25,10 +24,16 @@ from .test_postgresql import MockPostmaster
 
 
 def mock_import(*args, **kwargs):
-    if args[0] == 'psycopg':
+    ret = Mock()
+    ret.__version__ = '2.5.3.dev1 a b c' if args[0] == 'psycopg2' else '3.1.0'
+    return ret
+
+
+def mock_import2(*args, **kwargs):
+    if args[0] == 'psycopg2':
         raise ImportError
     ret = Mock()
-    ret.__version__ = '2.5.3.dev1 a b c'
+    ret.__version__ = '0.1.2'
     return ret
 
 
@@ -205,6 +210,8 @@ class TestPatroni(unittest.TestCase):
         with patch('builtins.__import__', Mock(side_effect=ImportError)):
             self.assertRaises(SystemExit, check_psycopg)
         with patch('builtins.__import__', mock_import):
+            self.assertIsNone(check_psycopg())
+        with patch('builtins.__import__', mock_import2):
             self.assertRaises(SystemExit, check_psycopg)
 
     def test_ensure_unique_name(self):