diff --git a/.travis.yml b/.travis.yml index 81c3f91b..a863eede 100644 --- a/.travis.yml +++ b/.travis.yml @@ -119,7 +119,7 @@ install: fi source ~/virtualenv/python${pv}/bin/activate # explicitly install all needed python modules to cache them - for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8 mock pytest-cov pytest setuptools'; do + for p in '-r requirements.txt' 'psycopg2-binary behave codacy-coverage coverage coveralls flake8 mock pytest-cov pytest setuptools'; do pip install $p --upgrade done fi diff --git a/README.rst b/README.rst index 9dd9c16f..0cc2baaf 100644 --- a/README.rst +++ b/README.rst @@ -59,6 +59,33 @@ To install requirements on a Mac, run the following: brew install postgresql etcd haproxy libyaml python +**Psycopg2** + +Starting from `psycopg2-2.8 `__ the binary version of psycopg2 will no longer be installed by default. Installing it from the source code requires C compiler and postgres+python dev packages. +Since in the python world it is not possible to specify dependency as ``psycopg2 OR psycopg2-binary`` you will have to decide how to install it. + +There are a few options available: + +1. Use the package manager from your distro + +:: + + sudo apt-get install python-psycopg2 # install python2 psycopg2 module on Debian/Ubuntu + sudo apt-get install python3-psycopg2 # install python3 psycopg2 module on Debian/Ubuntu + sudo yum install python-psycopg2 # install python2 psycopg2 on RedHat/Fedora/CentOS + +2. Install psycopg2 from the binary package + +:: + + pip install psycopg2-binary + +3. Install psycopg2 from source + +:: + + pip install psycopg2>=2.5.4 + **General installation for pip** Patroni can be installed with pip: diff --git a/docs/README.rst b/docs/README.rst index 345ed11f..8e0e3a7e 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -33,6 +33,33 @@ To install requirements on a Mac, run the following: brew install postgresql etcd haproxy libyaml python +**Psycopg2** + +Starting from `psycopg2-2.8 `__ the binary version of psycopg2 will no longer be installed by default. Installing it from the source code requires C compiler and postgres+python dev packages. +Since in the python world it is not possible to specify dependency as ``psycopg2 OR psycopg2-binary`` you will have to decide how to install it. + +There are a few options available: + +1. Use the package manager from your distro + +:: + + sudo apt-get install python-psycopg2 # install python2 psycopg2 module on Debian/Ubuntu + sudo apt-get install python3-psycopg2 # install python3 psycopg2 module on Debian/Ubuntu + sudo yum install python-psycopg2 # install python2 psycopg2 on RedHat/Fedora/CentOS + +2. Install psycopg2 from the binary package + +:: + + pip install psycopg2-binary + +3. Install psycopg2 from source + +:: + + pip install psycopg2>=2.5.4 + **General installation for pip** Patroni can be installed with pip: diff --git a/patroni/__init__.py b/patroni/__init__.py index c1e06073..6a0c5427 100644 --- a/patroni/__init__.py +++ b/patroni/__init__.py @@ -164,7 +164,27 @@ def patroni_main(): logging.shutdown() +def fatal(string, *args): + sys.stderr.write('FATAL: ' + string.format(*args) + '\n') + sys.exit(1) + + +def check_psycopg2(): + min_psycopg2 = (2, 5, 4) + min_psycopg2_str = '.'.join(map(str, min_psycopg2)) + + try: + import psycopg2 + version_str = psycopg2.__version__.split(' ')[0] + version = tuple(map(int, version_str.split('.'))) + if version < min_psycopg2: + fatal('Patroni requires psycopg2>={0}, but only {1} is available', min_psycopg2_str, version_str) + except ImportError: + fatal('Patroni requires psycopg2>={0} or psycopg2-binary', min_psycopg2_str) + + def main(): + check_psycopg2() if os.getpid() != 1: return patroni_main() diff --git a/patroni/ctl.py b/patroni/ctl.py index e2dcc5cc..bded4299 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -14,7 +14,6 @@ import io import json import logging import os -import psycopg2 import random import requests import subprocess @@ -246,6 +245,7 @@ def get_cursor(cluster, connect_parameters, role='master', member=None): else: params.pop('database') + import psycopg2 conn = psycopg2.connect(**params) conn.autocommit = True cursor = conn.cursor() @@ -380,6 +380,7 @@ def query( def query_member(cluster, cursor, member, role, command, connect_parameters): + import psycopg2 try: if cursor is None: cursor = get_cursor(cluster, connect_parameters, role=role, member=member) diff --git a/requirements.txt b/requirements.txt index 601358c1..17a3aa8b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ urllib3>=1.19.1,!=1.21 boto -psycopg2>=2.5.4 PyYAML requests six >= 1.7 diff --git a/setup.py b/setup.py index d845a751..027f5db0 100644 --- a/setup.py +++ b/setup.py @@ -8,27 +8,22 @@ import inspect import os import sys +from patroni import check_psycopg2, fatal +from patroni.version import __version__ as VERSION from setuptools.command.test import test as TestCommand from setuptools import find_packages, setup if sys.version_info < (2, 7, 0): - sys.stderr.write('FATAL: patroni needs to be run with Python 2.7+\n') - sys.exit(1) + fatal('patroni needs to be run with Python 2.7+') +check_psycopg2() +del sys.modules['patroni'] +del sys.modules['patroni.version'] __location__ = os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe()))) - -def read_version(package): - data = {} - with open(os.path.join(package, 'version.py'), 'r') as fd: - exec(fd.read(), data) - return data['__version__'] - - NAME = 'patroni' MAIN_PACKAGE = NAME SCRIPTS = 'scripts' -VERSION = read_version(MAIN_PACKAGE) DESCRIPTION = 'PostgreSQL High-Available orchestrator and CLI' LICENSE = 'The MIT License' URL = 'https://github.com/zalando/patroni' @@ -113,27 +108,23 @@ class PyTest(TestCommand): sys.exit(errno) -def get_install_requirements(path): - content = open(os.path.join(__location__, path)).read() - return [req for req in content.split('\n') if req != ''] - - def read(fname): - return open(os.path.join(__location__, fname)).read() + with open(os.path.join(__location__, fname)) as fd: + return fd.read() def setup_package(): # Assemble additional setup commands cmdclass = {'test': PyTest} - # Some helper variables - version = os.getenv('GO_PIPELINE_LABEL', VERSION) - install_requires = [] extras_require = {'aws': ['boto'], 'etcd': ['python-etcd'], 'consul': ['python-consul'], 'exhibitor': ['kazoo'], 'zookeeper': ['kazoo'], 'kubernetes': ['kubernetes']} - for r in get_install_requirements('requirements.txt'): + for r in read('requirements.txt').split('\n'): + r = r.strip() + if r == '': + continue extra = False for e, v in extras_require.items(): if r.startswith(v[0]): @@ -152,7 +143,7 @@ def setup_package(): setup( name=NAME, - version=version, + version=VERSION, url=URL, author=AUTHOR, author_email=AUTHOR_EMAIL, diff --git a/tests/test_api.py b/tests/test_api.py index 721a3932..d1d60c27 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -134,7 +134,6 @@ class MockRestApiServer(RestApiServer): def __init__(self, Handler, request, config=None): self.socket = 0 self.serve_forever = Mock() - BaseHTTPServer.HTTPServer.__init__ = Mock() MockRestApiServer._BaseServer__is_shut_down = Mock() MockRestApiServer._BaseServer__shutdown_request = True config = config or {'listen': '127.0.0.1:8008', 'auth': 'test:test', 'certfile': 'dumb'} @@ -143,6 +142,7 @@ class MockRestApiServer(RestApiServer): @patch('ssl.wrap_socket', Mock(return_value=0)) +@patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock()) class TestRestApiHandler(unittest.TestCase): _authorization = '\nAuthorization: Basic dGVzdDp0ZXN0' @@ -392,6 +392,7 @@ class TestRestApiHandler(unittest.TestCase): @patch('ssl.wrap_socket', Mock(return_value=0)) +@patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock()) class TestRestApiServer(unittest.TestCase): def test_reload_config(self): diff --git a/tests/test_patroni.py b/tests/test_patroni.py index 2406279d..9a4613e6 100644 --- a/tests/test_patroni.py +++ b/tests/test_patroni.py @@ -1,4 +1,5 @@ import etcd +import psycopg2 import signal import sys import time @@ -9,8 +10,8 @@ from patroni.api import RestApiServer from patroni.async_executor import AsyncExecutor from patroni.dcs.etcd import Client from patroni.exceptions import DCSError -from patroni import Patroni, main as _main, patroni_main -from six.moves import BaseHTTPServer +from patroni import Patroni, main as _main, patroni_main, check_psycopg2 +from six.moves import BaseHTTPServer, builtins from test_etcd import SleepException, etcd_read, etcd_write from test_postgresql import Postgresql, psycopg2_connect, MockPostmaster @@ -36,6 +37,7 @@ class TestPatroni(unittest.TestCase): @patch('pkgutil.get_importer', Mock(return_value=MockFrozenImporter())) @patch('sys.frozen', Mock(return_value=True), create=True) + @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock()) @patch.object(etcd.Client, 'read', etcd_read) def setUp(self): RestApiServer._BaseServer__is_shut_down = Mock() @@ -153,3 +155,9 @@ class TestPatroni(unittest.TestCase): def test_shutdown(self): self.p.api.shutdown = Mock(side_effect=Exception) self.p.shutdown() + + def test_check_psycopg2(self): + with patch.object(builtins, '__import__', Mock(side_effect=ImportError)): + self.assertRaises(SystemExit, check_psycopg2) + with patch.object(psycopg2, '__version__', return_value='2.5.3 a b c'): + self.assertRaises(SystemExit, check_psycopg2)