From e86cf9a722d1da399f6acdd6afd574251f9302ee Mon Sep 17 00:00:00 2001 From: Dmytro Aleksandrov Date: Mon, 1 Aug 2016 10:16:13 +0300 Subject: [PATCH 1/3] Attempt to make single binary build with pyinstaller --- .gitignore | 1 + mkbinary.sh | 5 +++++ patroni.spec | 29 +++++++++++++++++++++++++++++ patroni/config.py | 2 +- patroni/dcs/__init__.py | 41 +++++++++++++++++++++++++++-------------- requirements-bin.txt | 2 ++ 6 files changed, 65 insertions(+), 15 deletions(-) create mode 100755 mkbinary.sh create mode 100644 patroni.spec create mode 100644 requirements-bin.txt diff --git a/.gitignore b/.gitignore index fbb294d4..c03acb2b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ data/* .coverage .eggs/ build/ +dist/ coverage.xml junit.xml pgpass diff --git a/mkbinary.sh b/mkbinary.sh new file mode 100755 index 00000000..67829074 --- /dev/null +++ b/mkbinary.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -e + +pip install --ignore-installed -r requirements-bin.txt +pyinstaller --clean --onefile patroni.spec diff --git a/patroni.spec b/patroni.spec new file mode 100644 index 00000000..8c5c5347 --- /dev/null +++ b/patroni.spec @@ -0,0 +1,29 @@ +# -*- mode: python -*- + +block_cipher = None + + +a = Analysis(['patroni/__main__.py', 'patroni/dcs/consul.py', 'patroni/dcs/etcd.py', 'patroni/dcs/exhibitor.py', 'patroni/dcs/zookeeper.py'], + pathex=[], + binaries=None, + datas=None, + hiddenimports=['patroni.dcs.consul', 'patroni.dcs.etcd', 'patroni.dcs.exhibitor', 'patroni.dcs.zookeeper'], + hookspath=[], + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher) + +pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) + +exe = EXE(pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + name='patroni', + debug=False, + strip=False, + upx=True, + console=True) diff --git a/patroni/config.py b/patroni/config.py index f09cfca4..69757067 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -64,7 +64,7 @@ class Config(object): print('Usage: {0} config.yml'.format(sys.argv[0])) print('\tPatroni may also read the configuration from the {0} environment variable'. format(self.PATRONI_CONFIG_VARIABLE)) - exit(1) + sys.exit(1) self.__effective_configuration = self._build_effective_configuration({}, self._local_configuration) self._data_dir = self.__effective_configuration['postgresql']['data_dir'] diff --git a/patroni/dcs/__init__.py b/patroni/dcs/__init__.py index d7a69997..0045083c 100644 --- a/patroni/dcs/__init__.py +++ b/patroni/dcs/__init__.py @@ -6,6 +6,7 @@ import json import os import pkgutil import six +import sys from collections import namedtuple from patroni.exceptions import PatroniException @@ -31,22 +32,34 @@ def parse_connection_string(value): return conn_url, api_url +def dcs_modules(): + """Get names of DCS modules, depending on execution environment. If being packaged with PyInstaller, + modules aren't discoverable dynamically by scanning source directory. Thus, when running in bundle, + a predefined list of dcs modules is returned. See: + https://pyinstaller.readthedocs.io/en/stable/runtime-information.html#run-time-information""" + + if getattr(sys, 'frozen', False): + return ['consul', 'etcd', 'zookeeper', 'exhibitor'] + else: + module_names = (name for _, name, is_pkg in pkgutil.iter_modules([os.path.dirname(__file__)]) if not is_pkg) + return module_names + + def get_dcs(config): available_implementations = set() - for _, module_name, is_pkg in pkgutil.iter_modules([os.path.dirname(__file__)]): - if not is_pkg: - module = importlib.import_module(__package__ + '.' + module_name) - for name in filter(lambda name: not name.startswith('__'), dir(module)): # iterate through module content - value = getattr(module, name) - name = name.lower() - # try to find implementation of AbstractDCS interface, class name must match with module_name - if inspect.isclass(value) and issubclass(value, AbstractDCS) and name == module_name: - available_implementations.add(name) - if name in config: # which has configuration section in the config file - # propagate some parameters - config[name].update({p: config[p] for p in ('namespace', 'name', 'scope', - 'loop_wait', 'ttl', 'retry_timeout') if p in config}) - return value(config[name]) + for module_name in dcs_modules(): + module = importlib.import_module(__package__ + '.' + module_name) + for name in filter(lambda name: not name.startswith('__'), dir(module)): # iterate through module content + value = getattr(module, name) + name = name.lower() + # try to find implementation of AbstractDCS interface, class name must match with module_name + if inspect.isclass(value) and issubclass(value, AbstractDCS) and name == module_name: + available_implementations.add(name) + if name in config: # which has configuration section in the config file + # propagate some parameters + config[name].update({p: config[p] for p in ('namespace', 'name', 'scope', + 'loop_wait', 'ttl', 'retry_timeout') if p in config}) + return value(config[name]) raise PatroniException("""Can not find suitable configuration of distributed configuration store Available implementations: """ + ', '.join(available_implementations)) diff --git a/requirements-bin.txt b/requirements-bin.txt new file mode 100644 index 00000000..7e43a3fd --- /dev/null +++ b/requirements-bin.txt @@ -0,0 +1,2 @@ +setuptools==19.2 +pyinstaller From 5c7efa3a65dce0155887e3f1334412b46875730a Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 5 Sep 2016 14:10:21 +0200 Subject: [PATCH 2/3] Update gitignore --- .gitignore | 49 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index c03acb2b..236f02ea 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,48 @@ -data/* -*.pyc -*.egg/ -*.egg-info/ +*.py[cod] + +# vi(m) swap files: +*.sw? + +# C extensions +*.so + +# Packages .cache/ +*.egg +*.eggs +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports .coverage -.eggs/ -build/ -dist/ +.tox +nosetests.xml coverage.xml +htmlcov junit.xml +features/output +dummy + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + pgpass scm-source.json From 2086c90a4a32f25ce2ff6aff32a21592f6b47be0 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 5 Sep 2016 14:11:53 +0200 Subject: [PATCH 3/3] Try to get rid from hardcoded names when building binary --- mkbinary.sh | 2 +- patroni.spec | 14 ++++++++++++-- patroni/dcs/__init__.py | 19 +++++++++++-------- requirements-bin.txt | 2 -- tests/test_patroni.py | 7 +++++++ 5 files changed, 31 insertions(+), 13 deletions(-) delete mode 100644 requirements-bin.txt diff --git a/mkbinary.sh b/mkbinary.sh index 67829074..86cf8215 100755 --- a/mkbinary.sh +++ b/mkbinary.sh @@ -1,5 +1,5 @@ #!/bin/sh set -e -pip install --ignore-installed -r requirements-bin.txt +pip install --ignore-installed setuptools==19.2 pyinstaller pyinstaller --clean --onefile patroni.spec diff --git a/patroni.spec b/patroni.spec index 8c5c5347..2afe8eac 100644 --- a/patroni.spec +++ b/patroni.spec @@ -3,11 +3,21 @@ block_cipher = None -a = Analysis(['patroni/__main__.py', 'patroni/dcs/consul.py', 'patroni/dcs/etcd.py', 'patroni/dcs/exhibitor.py', 'patroni/dcs/zookeeper.py'], +def hiddenimports(): + import sys + sys.path.insert(0, '.') + try: + import patroni.dcs + return patroni.dcs.dcs_modules() + finally: + sys.path.pop(0) + + +a = Analysis(['patroni/__main__.py'], pathex=[], binaries=None, datas=None, - hiddenimports=['patroni.dcs.consul', 'patroni.dcs.etcd', 'patroni.dcs.exhibitor', 'patroni.dcs.zookeeper'], + hiddenimports=hiddenimports(), hookspath=[], runtime_hooks=[], excludes=[], diff --git a/patroni/dcs/__init__.py b/patroni/dcs/__init__.py index 0045083c..733efbb0 100644 --- a/patroni/dcs/__init__.py +++ b/patroni/dcs/__init__.py @@ -34,26 +34,29 @@ def parse_connection_string(value): def dcs_modules(): """Get names of DCS modules, depending on execution environment. If being packaged with PyInstaller, - modules aren't discoverable dynamically by scanning source directory. Thus, when running in bundle, - a predefined list of dcs modules is returned. See: - https://pyinstaller.readthedocs.io/en/stable/runtime-information.html#run-time-information""" + modules aren't discoverable dynamically by scanning source directory because `FrozenImporter` doesn't + implement `iter_modules` method. But it is still possible to find all potential DCS modules by + iterating through `toc`, which contains list of all "frozen" resources.""" + + dcs_dirname = os.path.dirname(__file__) + module_prefix = __package__ + '.' if getattr(sys, 'frozen', False): - return ['consul', 'etcd', 'zookeeper', 'exhibitor'] + importer = pkgutil.get_importer(dcs_dirname) + return [module for module in list(importer.toc) if module.startswith(module_prefix) and module.count('.') == 2] else: - module_names = (name for _, name, is_pkg in pkgutil.iter_modules([os.path.dirname(__file__)]) if not is_pkg) - return module_names + return [module_prefix + name for _, name, is_pkg in pkgutil.iter_modules([dcs_dirname]) if not is_pkg] def get_dcs(config): available_implementations = set() for module_name in dcs_modules(): - module = importlib.import_module(__package__ + '.' + module_name) + module = importlib.import_module(module_name) for name in filter(lambda name: not name.startswith('__'), dir(module)): # iterate through module content value = getattr(module, name) name = name.lower() # try to find implementation of AbstractDCS interface, class name must match with module_name - if inspect.isclass(value) and issubclass(value, AbstractDCS) and name == module_name: + if inspect.isclass(value) and issubclass(value, AbstractDCS) and __package__ + '.' + name == module_name: available_implementations.add(name) if name in config: # which has configuration section in the config file # propagate some parameters diff --git a/requirements-bin.txt b/requirements-bin.txt deleted file mode 100644 index 7e43a3fd..00000000 --- a/requirements-bin.txt +++ /dev/null @@ -1,2 +0,0 @@ -setuptools==19.2 -pyinstaller diff --git a/tests/test_patroni.py b/tests/test_patroni.py index f9f135cc..d9c0e93a 100644 --- a/tests/test_patroni.py +++ b/tests/test_patroni.py @@ -14,6 +14,11 @@ from test_etcd import SleepException, etcd_read, etcd_write from test_postgresql import Postgresql, psycopg2_connect +class MockFrozenImporter(object): + + toc = set(['patroni.dcs.etcd']) + + @patch('time.sleep', Mock()) @patch('subprocess.call', Mock(return_value=0)) @patch('psycopg2.connect', psycopg2_connect) @@ -27,6 +32,8 @@ from test_postgresql import Postgresql, psycopg2_connect @patch.object(etcd.Client, 'read', etcd_read) class TestPatroni(unittest.TestCase): + @patch('pkgutil.get_importer', Mock(return_value=MockFrozenImporter())) + @patch('sys.frozen', Mock(return_value=True), create=True) @patch.object(etcd.Client, 'read', etcd_read) def setUp(self): RestApiServer._BaseServer__is_shut_down = Mock()