diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 77a4df5b..7a102379 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -24,8 +24,11 @@ jobs: - name: Run tests and flake8 run: python .github/workflows/run_tests.py + - name: Install Python packaging build frontend + run: python -m pip install build + - name: Build a binary wheel and a source tarball - run: python setup.py sdist bdist_wheel + run: python -m build - name: Publish distribution to Test PyPI if: github.event_name == 'push' diff --git a/README.rst b/README.rst index decc868f..c2fc165e 100644 --- a/README.rst +++ b/README.rst @@ -77,23 +77,8 @@ There are a few options available: sudo apt-get install python3-psycopg2 # install psycopg2 module on Debian/Ubuntu sudo yum install python3-psycopg2 # install psycopg2 on RedHat/Fedora/CentOS -2. Install psycopg2 from the binary package +2. Specify one of `psycopg`, `psycopg2`, or `psycopg2-binary` in the list of dependencies when installing Patroni with pip (see below). -:: - - pip install psycopg2-binary - -3. Install psycopg2 from source - -:: - - pip install psycopg2>=2.5.4 - -4. Use psycopg 3.0 instead of psycopg2 - -:: - - pip install psycopg[binary]>=3.0.0 **General installation for pip** @@ -119,12 +104,20 @@ raft `pysyncobj` module in order to use python Raft implementation as DCS aws `boto3` in order to use AWS callbacks +all + all of the above (except psycopg family) +psycopg3 + `psycopg[binary]>=3.0.0` module +psycopg2 + `psycopg2>=2.5.4` module +psycopg2-binary + `psycopg2-binary` module -For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is: +For example, the command in order to install Patroni together with psycopg3, dependencies for Etcd as a DCS, and AWS callbacks is: :: - pip install patroni[etcd,aws] + pip install patroni[psycopg3,etcd3,aws] Note that external tools to call in the replica creation or custom bootstrap scripts (i.e. WAL-E) should be installed independently of Patroni. diff --git a/docs/installation.rst b/docs/installation.rst index 6c7a6029..3c9dddfa 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -30,23 +30,10 @@ There are a few options available: sudo apt-get install python3-psycopg2 # install psycopg2 module on Debian/Ubuntu sudo yum install python3-psycopg2 # install psycopg2 on RedHat/Fedora/CentOS -2. Install psycopg2 from the binary package +2. Specify one of `psycopg`, `psycopg2`, or `psycopg2-binary` in the :ref:`list of dependencies ` when installing Patroni with pip. -.. code-block:: shell - pip install psycopg2-binary - -3. Install psycopg2 from source - -.. code-block:: shell - - pip install psycopg2>=2.5.4 - -4. Use psycopg 3.0 instead of psycopg2 - -.. code-block:: shell - - pip install psycopg[binary]>=3.0.0 +.. _extras: General installation for pip ---------------------------- @@ -73,12 +60,20 @@ raft `pysyncobj` module in order to use python Raft implementation as DCS aws `boto3` in order to use AWS callbacks +all + all of the above (except psycopg family) +psycopg + `psycopg[binary]>=3.0.0` module +psycopg2 + `psycopg2>=2.5.4` module +psycopg2-binary + `psycopg2-binary` module -For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is: +For example, the command in order to install Patroni together with psycopg3, dependencies for Etcd as a DCS, and AWS callbacks is: .. code-block:: shell - pip install patroni[etcd,aws] + pip install patroni[psycopg3,etcd3,aws] Note that external tools to call in the replica creation or custom bootstrap scripts (i.e. WAL-E) should be installed independently of Patroni. diff --git a/patroni/__init__.py b/patroni/__init__.py index 7f7035c2..7e67e299 100644 --- a/patroni/__init__.py +++ b/patroni/__init__.py @@ -3,23 +3,14 @@ :var PATRONI_ENV_PREFIX: prefix for Patroni related configuration environment variables. :var KUBERNETES_ENV_PREFIX: prefix for Kubernetes related configuration environment variables. :var MIN_PSYCOPG2: minimum version of :mod:`psycopg2` required by Patroni to work. +:var MIN_PSYCOPG3: minimum version of :mod:`psycopg` required by Patroni to work. """ - -import sys - -from typing import Any, Callable, Iterator, Tuple +from typing import Iterator, Tuple PATRONI_ENV_PREFIX = 'PATRONI_' KUBERNETES_ENV_PREFIX = 'KUBERNETES_' MIN_PSYCOPG2 = (2, 5, 4) - - -def fatal(string: str, *args: Any) -> None: - """Write a fatal message to stderr and exit with code ``1``. - - :param string: message to be written before exiting. - """ - sys.exit('FATAL: ' + string.format(*args)) +MIN_PSYCOPG3 = (3, 0, 0) def parse_version(version: str) -> Tuple[int, ...]: @@ -28,25 +19,25 @@ def parse_version(version: str) -> Tuple[int, ...]: .. note:: Designed for easy comparison of software versions in Python. - :param version: human-readable software version, e.g. ``2.5.4``. + :param version: human-readable software version, e.g. ``2.5.4.dev1 (dt dec pq3 ext lo64)``. :returns: tuple of *version* parts, each part as an integer. :Example: - >>> parse_version('2.5.4') + >>> parse_version('2.5.4.dev1 (dt dec pq3 ext lo64)') (2, 5, 4) """ def _parse_version(version: str) -> Iterator[int]: """Yield each part of a human-readable version string as an integer. - :param version: human-readable software version, e.g. ``2.5.4``. + :param version: human-readable software version, e.g. ``2.5.4.dev1``. :yields: each part of *version* as an integer. :Example: - >>> tuple(_parse_version('2.5.4')) + >>> tuple(_parse_version('2.5.4.dev1')) (2, 5, 4) """ for e in version.split('.'): @@ -55,40 +46,3 @@ def parse_version(version: str) -> Tuple[int, ...]: except ValueError: break return tuple(_parse_version(version.split(' ')[0])) - - -def check_psycopg(_min_psycopg2: Tuple[int, ...] = MIN_PSYCOPG2, - _parse_version: Callable[[str], Tuple[int, ...]] = parse_version) -> None: - """Ensure at least one among :mod:`psycopg2` or :mod:`psycopg` libraries are available in the environment. - - .. note:: - We pass ``MIN_PSYCOPG2`` and :func:`parse_version` as arguments to simplify usage of :func:`check_psycopg` from - the ``setup.py``. - - .. note:: - Patroni chooses :mod:`psycopg2` over :mod:`psycopg`, if possible. - - If nothing meeting the requirements is found, then exit with a fatal message. - - :param _min_psycopg2: minimum required version in case :mod:`psycopg2` is chosen. - :param _parse_version: function used to parse :mod:`psycopg2`/:mod:`psycopg` version into a comparable object. - """ - min_psycopg2_str = '.'.join(map(str, _min_psycopg2)) - - # try psycopg2 - try: - from psycopg2 import __version__ - if _parse_version(__version__) >= _min_psycopg2: - return - version_str = __version__.split(' ')[0] - except ImportError: - version_str = None - - # try psycopg3 - try: - from psycopg import __version__ - except ImportError: - error = 'Patroni requires psycopg2>={0}, psycopg2-binary, or psycopg>=3.0'.format(min_psycopg2_str) - if version_str is not None: - error += ', but only psycopg2=={0} is available'.format(version_str) - fatal(error) diff --git a/patroni/__main__.py b/patroni/__main__.py index 7d56172b..02ba56da 100644 --- a/patroni/__main__.py +++ b/patroni/__main__.py @@ -10,8 +10,9 @@ import sys import time from argparse import Namespace -from typing import Any, Dict, Optional, TYPE_CHECKING +from typing import Any, Dict, List, Optional, TYPE_CHECKING +from patroni import MIN_PSYCOPG2, MIN_PSYCOPG3, parse_version from patroni.daemon import AbstractPatroniDaemon, abstract_main, get_base_arg_parser from patroni.tags import Tags @@ -286,6 +287,45 @@ def process_arguments() -> Namespace: return args +def check_psycopg() -> None: + """Ensure at least one among :mod:`psycopg2` or :mod:`psycopg` libraries are available in the environment. + + .. note:: + Patroni chooses :mod:`psycopg2` over :mod:`psycopg`, if possible. + + If nothing meeting the requirements is found, then exit with a fatal message. + """ + min_psycopg2_str = '.'.join(map(str, MIN_PSYCOPG2)) + min_psycopg3_str = '.'.join(map(str, MIN_PSYCOPG3)) + + available_versions: List[str] = [] + + # try psycopg2 + try: + from psycopg2 import __version__ + if parse_version(__version__) >= MIN_PSYCOPG2: + return + available_versions.append('psycopg2=={0}'.format(__version__.split(' ')[0])) + except ImportError: + logger.debug('psycopg2 module is not available') + + # try psycopg3 + try: + from psycopg import __version__ + if parse_version(__version__) >= MIN_PSYCOPG3: + return + available_versions.append('psycopg=={0}'.format(__version__.split(' ')[0])) + except ImportError: + logger.debug('psycopg module is not available') + + error = f'FATAL: Patroni requires psycopg2>={min_psycopg2_str}, psycopg2-binary, or psycopg>={min_psycopg3_str}' + if available_versions: + error += ', but only {0} {1} available'.format( + ' and '.join(available_versions), + 'is' if len(available_versions) == 1 else 'are') + sys.exit(error) + + def main() -> None: """Main entrypoint of :mod:`patroni.__main__`. @@ -297,12 +337,10 @@ def main() -> None: ``patroni`` daemon as another process. In that case relevant signals received by the main process and forwarded to ``patroni`` daemon process. """ - from patroni import check_psycopg + check_psycopg() args = process_arguments() - check_psycopg() - if os.getpid() != 1: return patroni_main(args.configfile) diff --git a/setup.py b/setup.py index d61eab6e..4c1c25b3 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,6 @@ KEYWORDS = 'etcd governor patroni postgresql postgres ha haproxy confd' +\ EXTRAS_REQUIRE = {'aws': ['boto3'], 'etcd': ['python-etcd'], 'etcd3': ['python-etcd'], 'consul': ['python-consul'], 'exhibitor': ['kazoo'], 'zookeeper': ['kazoo'], 'kubernetes': [], 'raft': ['pysyncobj', 'cryptography']} -COVERAGE_XML = True # Add here all kinds of additional classifiers as defined under # https://pypi.python.org/pypi?%3Aaction=list_classifiers @@ -120,14 +119,21 @@ def read(fname): return fd.read() -def setup_package(version): +def get_versions(): + old_modules = sys.modules.copy() + try: + from patroni import MIN_PSYCOPG2, MIN_PSYCOPG3 + from patroni.version import __version__ + return __version__, MIN_PSYCOPG2, MIN_PSYCOPG3 + finally: + sys.modules.clear() + sys.modules.update(old_modules) + + +def main(): logging.basicConfig(format='%(message)s', level=os.getenv('LOGLEVEL', logging.WARNING)) - # Assemble additional setup commands - cmdclass = {'test': PyTest, 'flake8': Flake8} - install_requires = [] - for r in read('requirements.txt').split('\n'): r = r.strip() if r == '': @@ -139,15 +145,22 @@ def setup_package(version): deps[i] = r EXTRAS_REQUIRE[e] = deps extra = True - break - if extra: - break if not extra: install_requires.append(r) + # Just for convenience, if someone wants to install dependencies for all extras + EXTRAS_REQUIRE['all'] = list({e for extras in EXTRAS_REQUIRE.values() for e in extras}) + + patroni_version, min_psycopg2, min_psycopg3 = get_versions() + + # Make it possible to specify psycopg dependency as extra + for name, version in {'psycopg[binary]': min_psycopg3, 'psycopg2': min_psycopg2, 'psycopg2-binary': None}.items(): + EXTRAS_REQUIRE[name] = [name + ('>=' + '.'.join(map(str, version)) if version else '')] + EXTRAS_REQUIRE['psycopg3'] = EXTRAS_REQUIRE.pop('psycopg[binary]') + setup( name=NAME, - version=version, + version=patroni_version, url=URL, author=AUTHOR, author_email=AUTHOR_EMAIL, @@ -163,20 +176,10 @@ def setup_package(version): ]}, install_requires=install_requires, extras_require=EXTRAS_REQUIRE, - cmdclass=cmdclass, + cmdclass={'test': PyTest, 'flake8': Flake8}, entry_points={'console_scripts': CONSOLE_SCRIPTS}, ) if __name__ == '__main__': - old_modules = sys.modules.copy() - try: - from patroni import check_psycopg - from patroni.version import __version__ - finally: - sys.modules.clear() - sys.modules.update(old_modules) - - check_psycopg() - - setup_package(__version__) + main() diff --git a/tests/test_patroni.py b/tests/test_patroni.py index df59677d..137157e5 100644 --- a/tests/test_patroni.py +++ b/tests/test_patroni.py @@ -15,8 +15,7 @@ from patroni.dcs.etcd import AbstractEtcdClientWithFailover from patroni.exceptions import DCSError from patroni.postgresql import Postgresql from patroni.postgresql.config import ConfigHandler -from patroni import check_psycopg -from patroni.__main__ import Patroni, main as _main +from patroni.__main__ import check_psycopg, Patroni, main as _main from threading import Thread from . import psycopg_connect, SleepException @@ -25,10 +24,16 @@ from .test_postgresql import MockPostmaster def mock_import(*args, **kwargs): - if args[0] == 'psycopg': + ret = Mock() + ret.__version__ = '2.5.3.dev1 a b c' if args[0] == 'psycopg2' else '3.1.0' + return ret + + +def mock_import2(*args, **kwargs): + if args[0] == 'psycopg2': raise ImportError ret = Mock() - ret.__version__ = '2.5.3.dev1 a b c' + ret.__version__ = '0.1.2' return ret @@ -205,6 +210,8 @@ class TestPatroni(unittest.TestCase): with patch('builtins.__import__', Mock(side_effect=ImportError)): self.assertRaises(SystemExit, check_psycopg) with patch('builtins.__import__', mock_import): + self.assertIsNone(check_psycopg()) + with patch('builtins.__import__', mock_import2): self.assertRaises(SystemExit, check_psycopg) def test_ensure_unique_name(self):