first commit

Close #772, #1128

docker/README.md

@@ -67,24 +67,24 @@ Example session:
 
     $ docker exec -ti demo-patroni1 bash
     postgres@patroni1:~$ patronictl list
-    +-------------+----------+------------+--------+---------+----+-----------+
+    +---------+----------+------------+--------+---------+----+-----------+
-    |   Cluster   |  Member  |    Host    |  Role  |  State  | TL | Lag in MB |
+    | Cluster |  Member  |    Host    |  Role  |  State  | TL | Lag in MB |
-    +-------------+----------+------------+--------+---------+----+-----------+
+    +---------+----------+------------+--------+---------+----+-----------+
-    | testcluster | patroni1 | 172.21.0.3 | Leader | running |  1 |         0 |
+    |   demo  | patroni1 | 172.22.0.3 | Leader | running |  1 |         0 |
-    | testcluster | patroni2 | 172.21.0.4 |        | running |  1 |         0 |
+    |   demo  | patroni2 | 172.22.0.7 |        | running |  1 |         0 |
-    | testcluster | patroni3 | 172.21.0.5 |        | running |  1 |         0 |
+    |   demo  | patroni3 | 172.22.0.4 |        | running |  1 |         0 |
-    +-------------+----------+------------+--------+---------+----+-----------+
+    +---------+----------+------------+--------+---------+----+-----------+
 
-    postgres@patroni1:~$ etcdctl ls --recursive --sort -p /service/testcluster
+    postgres@patroni1:~$ etcdctl ls --recursive --sort -p /service/demo
-    /service/testcluster/config
+    /service/demo/config
-    /service/testcluster/initialize
+    /service/demo/initialize
-    /service/testcluster/leader
+    /service/demo/leader
-    /service/testcluster/members/
+    /service/demo/members/
-    /service/testcluster/members/patroni1
+    /service/demo/members/patroni1
-    /service/testcluster/members/patroni2
+    /service/demo/members/patroni2
-    /service/testcluster/members/patroni3
+    /service/demo/members/patroni3
-    /service/testcluster/optime/
+    /service/demo/optime/
-    /service/testcluster/optime/leader
+    /service/demo/optime/leader
 
     postgres@patroni1:~$ etcdctl member list
     1bab629f01fa9065: name=etcd3 peerURLs=http://etcd3:2380 clientURLs=http://etcd3:2379 isLeader=false

docs/ENVIRONMENT.rst

@@ -19,6 +19,8 @@ Global/Universal
 -  **PATRONI\_LOG\_FILE\_NUM**: The number of application logs to retain.
 -  **PATRONI\_LOG\_FILE\_SIZE**: Size of patroni.log file (in bytes) that triggers a log rolling.
 -  **PATRONI\_LOG\_LOGGERS**: Redefine logging level per python module. Example ``PATRONI_LOG_LOGGERS="{patroni.postmaster: WARNING, urllib3: DEBUG}"``
+-  **PATRONI\_DEBUG\_MODE**: When set to a non-empty value, makes Patroni run in the special debug mode that enables stepping with a debugger through some execution paths within Patroni `
+Example ``PATRONI_DEBUG_MODE="on"``
 
 Bootstrap configuration
 -----------------------

patroni.py

@@ -1,6 +1,14 @@
 #!/usr/bin/env python
 from patroni import main
 
+import os
 
 if __name__ == '__main__':
+
+    if os.getenv("PATRONI_DEBUG_MODE"):
+        # XXX Visual Code specific https://github.com/microsoft/ptvsd/issues/1443
+        # create processes by spawning new Python interpreters instead of forking the current one 
+        import multiprocessing
+        multiprocessing.set_start_method('spawn', True)
+
     main()

patroni/__init__.py

@@ -32,6 +32,7 @@ class Patroni(object):
         self.postgresql = Postgresql(self.config['postgresql'])
         self.api = RestApiServer(self, self.config['restapi'])
         self.ha = Ha(self)
+        self.is_in_debug_mode = bool(os.getenv("PATRONI_DEBUG_MODE"))
 
         self.tags = self.get_tags()
         self.next_run = time.time()
@@ -102,8 +103,9 @@ class Patroni(object):
             self.next_run = current_time
             # Release the GIL so we don't starve anyone waiting on async_executor lock
             time.sleep(0.001)
-            # Warn user that Patroni is not keeping up
+            # Warn user that Patroni is not keeping up or runs in debug
-            logger.warning("Loop time exceeded, rescheduling immediately.")
+            msg = "Patroni runs in the debug mode: keys' TTL is infinite, loop wait disabled" if self.is_in_debug_mode else "Loop time exceeded, rescheduling immediately."
+            logger.warning(msg)
         elif self.ha.watch(nap_time):
             self.next_run = time.time()
 

patroni/api.py

@@ -101,7 +101,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
         else:
             primary_status_code = 200 if patroni.ha.is_leader() else 503
 
-        replica_status_code = 200 if not patroni.noloadbalance and response.get('role') == 'replica' else 503
+        replica_status_code = 200 if not patroni.noloadbalance and \
+            response.get('role') == 'replica' and response.get('state') == 'running' else 503
         status_code = 503
 
         if patroni.ha.is_standby_cluster() and ('standby_leader' in path or 'standby-leader' in path):

patroni/config.py

@@ -333,6 +333,11 @@ class Config(object):
             elif name not in config or name in ['watchdog']:
                 config[name] = deepcopy(value) if value else {}
         
+        if os.getenv("PATRONI_DEBUG_MODE"):
+            config['ttl'] = 24 * 60 * 60 # practical infinity for a debugging session
+            config['loop_wait'] = -1
+
+
         # restapi server expects to get restapi.auth = 'username:password'
         if 'authentication' in config['restapi']:
             config['restapi']['auth'] = '{username}:{password}'.format(**config['restapi']['authentication'])

patroni/ctl.py

@@ -25,7 +25,6 @@ import yaml
 
 from click import ClickException
 from contextlib import contextmanager
-from distutils.spawn import find_executable
 from patroni.config import Config
 from patroni.dcs import get_dcs as _get_dcs
 from patroni.exceptions import PatroniException
@@ -111,7 +110,8 @@ option_insecure = click.option('-k', '--insecure', is_flag=True, help='Allow con
 
 
 @click.group()
-@click.option('--config-file', '-c', help='Configuration file', default=CONFIG_FILE_PATH)
+@click.option('--config-file', '-c', help='Configuration file',
+              envvar='PATRONICTL_CONFIG_FILE', default=CONFIG_FILE_PATH)
 @click.option('--dcs', '-d', help='Use this DCS', envvar='DCS')
 @option_insecure
 @click.pass_context
@@ -1106,6 +1106,24 @@ def apply_yaml_file(data, filename):
     return format_config_for_editing(changed_data), changed_data
 
 
+def find_executable(executable, path=None):
+    _, ext = os.path.splitext(executable)
+
+    if (sys.platform == 'win32') and (ext != '.exe'):
+        executable = executable + '.exe'
+
+    if os.path.isfile(executable):
+        return executable
+
+    if path is None:
+        path = os.environ.get('PATH', os.defpath)
+
+    for p in path.split(os.pathsep):
+        f = os.path.join(p, executable)
+        if os.path.isfile(f):
+            return f
+
+
 def invoke_editor(before_editing, cluster_name):
     """Starts editor command to edit configuration in human readable format
 

patroni/postgresql/bootstrap.py

@@ -344,8 +344,12 @@ END;$$""".format(name, ' '.join(options))
                     self.create_or_update_role(rewind['username'], rewind.get('password'), [])
                     for f in ('pg_ls_dir(text, boolean, boolean)', 'pg_stat_file(text, boolean)',
                               'pg_read_binary_file(text)', 'pg_read_binary_file(text, bigint, bigint, boolean)'):
-                        postgresql.query('GRANT EXECUTE ON function pg_catalog.{0} TO "{1}"'
+                        sql = """DO $$
-                                         .format(f, rewind['username']))
+BEGIN
+    SET local synchronous_commit = 'local';
+    GRANT EXECUTE ON function pg_catalog.{0} TO "{1}";
+END;$$""".format(f, rewind['username'])
+                        postgresql.query(sql)
 
                 for name, value in (config.get('users') or {}).items():
                     if all(name != a.get('username') for a in (superuser, replication, rewind)):

patroni/postgresql/config.py

@@ -176,14 +176,14 @@ class ConfigHandler(object):
         'cluster_name': (None, lambda _: False, 90500),
         'wal_level': ('hot_standby', lambda v: v.lower() in ('hot_standby', 'replica', 'logical'), 90100),
         'hot_standby': ('on', lambda _: False, 90100),
-        'max_connections': (100, lambda v: int(v) >= 100, 90100),
+        'max_connections': (100, lambda v: int(v) >= 25, 90100),
-        'max_wal_senders': (10, lambda v: int(v) >= 10, 90100),
+        'max_wal_senders': (10, lambda v: int(v) >= 3, 90100),
-        'wal_keep_segments': (8, lambda v: int(v) >= 8, 90100),
+        'wal_keep_segments': (8, lambda v: int(v) >= 1, 90100),
         'max_prepared_transactions': (0, lambda v: int(v) >= 0, 90100),
-        'max_locks_per_transaction': (64, lambda v: int(v) >= 64, 90100),
+        'max_locks_per_transaction': (64, lambda v: int(v) >= 32, 90100),
         'track_commit_timestamp': ('off', lambda v: parse_bool(v) is not None, 90500),
-        'max_replication_slots': (10, lambda v: int(v) >= 10, 90400),
+        'max_replication_slots': (10, lambda v: int(v) >= 4, 90400),
-        'max_worker_processes': (8, lambda v: int(v) >= 8, 90400),
+        'max_worker_processes': (8, lambda v: int(v) >= 2, 90400),
         'wal_log_hints': ('on', lambda _: False, 90400)
     })
 

patroni/postgresql/rewind.py

@@ -146,6 +146,7 @@ class Rewind(object):
     def pg_rewind(self, r):
         # prepare pg_rewind connection
         env = self._postgresql.write_pgpass(r)
+        env['PGOPTIONS'] = '-c statement_timeout=0'
         dsn_attrs = [
             ('user', r.get('user')),
             ('host', r.get('host')),

tests/test_ctl.py

@@ -9,7 +9,7 @@ from datetime import datetime, timedelta
 from mock import patch, Mock
 from patroni.ctl import ctl, store_config, load_config, output_members, request_patroni, get_dcs, parse_dcs, \
     get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException, apply_config_changes, \
-    format_config_for_editing, show_diff, invoke_editor, format_pg_version
+    format_config_for_editing, show_diff, invoke_editor, format_pg_version, find_executable
 from patroni.dcs.etcd import Client, Failover
 from patroni.utils import tzutc
 from psycopg2 import OperationalError
@@ -587,3 +587,12 @@ class TestCtl(unittest.TestCase):
     def test_format_pg_version(self):
         self.assertEqual(format_pg_version(100001), '10.1')
         self.assertEqual(format_pg_version(90605), '9.6.5')
+
+    @patch('sys.platform', 'win32')
+    def test_find_executable(self):
+        with patch('os.path.isfile', Mock(return_value=True)):
+            self.assertEqual(find_executable('vim'), 'vim.exe')
+        with patch('os.path.isfile', Mock(return_value=False)):
+            self.assertIsNone(find_executable('vim'))
+        with patch('os.path.isfile', Mock(side_effect=[False, True])):
+            self.assertEqual(find_executable('vim', '/'), '/vim.exe')