Remove psycopg2 from requirements (#1023)

Recently released psycopg2 split into two different packages, psycopg2, and psycopg2-binary which could be installed at the same time into the same place on the filesystem. In order to decrease dependency hell problem, we let a user choose how to install psycopg2. There are a few options available and it is reflected in the documentation.

This PR also changes the following behavior:
* `pip install patroni` will fail if psycopg2 is not installed
* Patroni will check psycopg2 upon start and fail if it can't be found or outdated.

Closes https://github.com/zalando/patroni/issues/1021

.travis.yml

@@ -119,7 +119,7 @@ install:
             fi
             source ~/virtualenv/python${pv}/bin/activate
             # explicitly install all needed python modules to cache them
-            for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8 mock pytest-cov pytest setuptools'; do
+            for p in '-r requirements.txt' 'psycopg2-binary behave codacy-coverage coverage coveralls flake8 mock pytest-cov pytest setuptools'; do
                 pip install $p --upgrade
             done
         fi

README.rst

@@ -59,6 +59,33 @@ To install requirements on a Mac, run the following:
 
     brew install postgresql etcd haproxy libyaml python
 
+**Psycopg2**
+
+Starting from `psycopg2-2.8 <http://initd.org/psycopg/articles/2019/04/04/psycopg-28-released/>`__ the binary version of psycopg2 will no longer be installed by default. Installing it from the source code requires C compiler and postgres+python dev packages.
+Since in the python world it is not possible to specify dependency as ``psycopg2 OR psycopg2-binary`` you will have to decide how to install it.
+
+There are a few options available:
+
+1. Use the package manager from your distro
+
+::
+
+    sudo apt-get install python-psycopg2   # install python2 psycopg2 module on Debian/Ubuntu
+    sudo apt-get install python3-psycopg2  # install python3 psycopg2 module on Debian/Ubuntu
+    sudo yum install python-psycopg2       # install python2 psycopg2 on RedHat/Fedora/CentOS
+
+2. Install psycopg2 from the binary package
+
+::
+
+    pip install psycopg2-binary
+
+3. Install psycopg2 from source
+
+::
+
+    pip install psycopg2>=2.5.4
+
 **General installation for pip**
 
 Patroni can be installed with pip:

docs/README.rst

@@ -33,6 +33,33 @@ To install requirements on a Mac, run the following:
 
     brew install postgresql etcd haproxy libyaml python
 
+**Psycopg2**
+
+Starting from `psycopg2-2.8 <http://initd.org/psycopg/articles/2019/04/04/psycopg-28-released/>`__ the binary version of psycopg2 will no longer be installed by default. Installing it from the source code requires C compiler and postgres+python dev packages.
+Since in the python world it is not possible to specify dependency as ``psycopg2 OR psycopg2-binary`` you will have to decide how to install it.
+
+There are a few options available:
+
+1. Use the package manager from your distro
+
+::
+
+    sudo apt-get install python-psycopg2   # install python2 psycopg2 module on Debian/Ubuntu
+    sudo apt-get install python3-psycopg2  # install python3 psycopg2 module on Debian/Ubuntu
+    sudo yum install python-psycopg2       # install python2 psycopg2 on RedHat/Fedora/CentOS
+
+2. Install psycopg2 from the binary package
+
+::
+
+    pip install psycopg2-binary
+
+3. Install psycopg2 from source
+
+::
+
+    pip install psycopg2>=2.5.4
+
 **General installation for pip**
 
 Patroni can be installed with pip:

patroni/__init__.py

@@ -164,7 +164,27 @@ def patroni_main():
         logging.shutdown()
 
 
+def fatal(string, *args):
+    sys.stderr.write('FATAL: ' + string.format(*args) + '\n')
+    sys.exit(1)
+
+
+def check_psycopg2():
+    min_psycopg2 = (2, 5, 4)
+    min_psycopg2_str = '.'.join(map(str, min_psycopg2))
+
+    try:
+        import psycopg2
+        version_str = psycopg2.__version__.split(' ')[0]
+        version = tuple(map(int, version_str.split('.')))
+        if version < min_psycopg2:
+            fatal('Patroni requires psycopg2>={0}, but only {1} is available', min_psycopg2_str, version_str)
+    except ImportError:
+        fatal('Patroni requires psycopg2>={0} or psycopg2-binary', min_psycopg2_str)
+
+
 def main():
+    check_psycopg2()
     if os.getpid() != 1:
         return patroni_main()
 

patroni/ctl.py

@@ -14,7 +14,6 @@ import io
 import json
 import logging
 import os
-import psycopg2
 import random
 import requests
 import subprocess
@@ -246,6 +245,7 @@ def get_cursor(cluster, connect_parameters, role='master', member=None):
     else:
         params.pop('database')
 
+    import psycopg2
     conn = psycopg2.connect(**params)
     conn.autocommit = True
     cursor = conn.cursor()
@@ -380,6 +380,7 @@ def query(
 
 
 def query_member(cluster, cursor, member, role, command, connect_parameters):
+    import psycopg2
     try:
         if cursor is None:
             cursor = get_cursor(cluster, connect_parameters, role=role, member=member)

requirements.txt

@@ -1,6 +1,5 @@
 urllib3>=1.19.1,!=1.21
 boto
-psycopg2>=2.5.4
 PyYAML
 requests
 six >= 1.7

setup.py

@@ -8,27 +8,22 @@ import inspect
 import os
 import sys
 
+from patroni import check_psycopg2, fatal
+from patroni.version import __version__ as VERSION
 from setuptools.command.test import test as TestCommand
 from setuptools import find_packages, setup
 
 if sys.version_info < (2, 7, 0):
-    sys.stderr.write('FATAL: patroni needs to be run with Python 2.7+\n')
+    fatal('patroni needs to be run with Python 2.7+')
-    sys.exit(1)
+check_psycopg2()
+del sys.modules['patroni']
+del sys.modules['patroni.version']
 
 __location__ = os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe())))
 
-
-def read_version(package):
-    data = {}
-    with open(os.path.join(package, 'version.py'), 'r') as fd:
-        exec(fd.read(), data)
-    return data['__version__']
-
-
 NAME = 'patroni'
 MAIN_PACKAGE = NAME
 SCRIPTS = 'scripts'
-VERSION = read_version(MAIN_PACKAGE)
 DESCRIPTION = 'PostgreSQL High-Available orchestrator and CLI'
 LICENSE = 'The MIT License'
 URL = 'https://github.com/zalando/patroni'
@@ -113,27 +108,23 @@ class PyTest(TestCommand):
         sys.exit(errno)
 
 
-def get_install_requirements(path):
-    content = open(os.path.join(__location__, path)).read()
-    return [req for req in content.split('\n') if req != '']
-
-
 def read(fname):
-    return open(os.path.join(__location__, fname)).read()
+    with open(os.path.join(__location__, fname)) as fd:
+        return fd.read()
 
 
 def setup_package():
     # Assemble additional setup commands
     cmdclass = {'test': PyTest}
 
-    # Some helper variables
-    version = os.getenv('GO_PIPELINE_LABEL', VERSION)
-
     install_requires = []
     extras_require = {'aws': ['boto'], 'etcd': ['python-etcd'], 'consul': ['python-consul'],
                       'exhibitor': ['kazoo'], 'zookeeper': ['kazoo'], 'kubernetes': ['kubernetes']}
 
-    for r in get_install_requirements('requirements.txt'):
+    for r in read('requirements.txt').split('\n'):
+        r = r.strip()
+        if r == '':
+            continue
         extra = False
         for e, v in extras_require.items():
             if r.startswith(v[0]):
@@ -152,7 +143,7 @@ def setup_package():
 
     setup(
         name=NAME,
-        version=version,
+        version=VERSION,
         url=URL,
         author=AUTHOR,
         author_email=AUTHOR_EMAIL,

tests/test_api.py

@@ -134,7 +134,6 @@ class MockRestApiServer(RestApiServer):
     def __init__(self, Handler, request, config=None):
         self.socket = 0
         self.serve_forever = Mock()
-        BaseHTTPServer.HTTPServer.__init__ = Mock()
         MockRestApiServer._BaseServer__is_shut_down = Mock()
         MockRestApiServer._BaseServer__shutdown_request = True
         config = config or {'listen': '127.0.0.1:8008', 'auth': 'test:test', 'certfile': 'dumb'}
@@ -143,6 +142,7 @@ class MockRestApiServer(RestApiServer):
 
 
 @patch('ssl.wrap_socket', Mock(return_value=0))
+@patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
 class TestRestApiHandler(unittest.TestCase):
 
     _authorization = '\nAuthorization: Basic dGVzdDp0ZXN0'
@@ -392,6 +392,7 @@ class TestRestApiHandler(unittest.TestCase):
 
 
 @patch('ssl.wrap_socket', Mock(return_value=0))
+@patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
 class TestRestApiServer(unittest.TestCase):
 
     def test_reload_config(self):

tests/test_patroni.py

@@ -1,4 +1,5 @@
 import etcd
+import psycopg2
 import signal
 import sys
 import time
@@ -9,8 +10,8 @@ from patroni.api import RestApiServer
 from patroni.async_executor import AsyncExecutor
 from patroni.dcs.etcd import Client
 from patroni.exceptions import DCSError
-from patroni import Patroni, main as _main, patroni_main
+from patroni import Patroni, main as _main, patroni_main, check_psycopg2
-from six.moves import BaseHTTPServer
+from six.moves import BaseHTTPServer, builtins
 from test_etcd import SleepException, etcd_read, etcd_write
 from test_postgresql import Postgresql, psycopg2_connect, MockPostmaster
 
@@ -36,6 +37,7 @@ class TestPatroni(unittest.TestCase):
 
     @patch('pkgutil.get_importer', Mock(return_value=MockFrozenImporter()))
     @patch('sys.frozen', Mock(return_value=True), create=True)
+    @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
     @patch.object(etcd.Client, 'read', etcd_read)
     def setUp(self):
         RestApiServer._BaseServer__is_shut_down = Mock()
@@ -153,3 +155,9 @@ class TestPatroni(unittest.TestCase):
     def test_shutdown(self):
         self.p.api.shutdown = Mock(side_effect=Exception)
         self.p.shutdown()
+
+    def test_check_psycopg2(self):
+        with patch.object(builtins, '__import__', Mock(side_effect=ImportError)):
+            self.assertRaises(SystemExit, check_psycopg2)
+        with patch.object(psycopg2, '__version__', return_value='2.5.3 a b c'):
+            self.assertRaises(SystemExit, check_psycopg2)