Bumped version to 1.1

Feature/disable automatic failover

.gitignore

@@ -1,12 +1,48 @@
-data/*
+*.py[cod]
-*.pyc
+
-*.egg/
+# vi(m) swap files:
-*.egg-info/
+*.sw?
+
+# C extensions
+*.so
+
+# Packages
 .cache/
+*.egg
+*.eggs
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
 .coverage
-.eggs/
+.tox
-build/
+nosetests.xml
 coverage.xml
+htmlcov
 junit.xml
+features/output
+dummy
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
 pgpass
 scm-source.json

.travis.yml

@@ -19,7 +19,7 @@ cache:
   directories:
   - $HOME/virtualenv/python2.7.9
   - $HOME/virtualenv/python3.4.2
-  - $HOME/virtualenv/python3.5.0
+  - $HOME/virtualenv/python3.5.2
 install:
   - |
     set -e
@@ -51,7 +51,7 @@ install:
     for pv in "2.7" "3.4" "3.5"; do
         source ~/virtualenv/python${pv}/bin/activate
         # explicitly install all needed python modules to cache them
-        for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8 mock>=2.0.0 pytest-cov pytest'; do
+        for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8==2.6.0 mock>=2.0.0 pytest-cov pytest'; do
             pip install $p
         done
     done

.zappr.yaml

@@ -0,0 +1,13 @@
+# for github.com
+approvals:
+  groups:
+    zalando:
+      minimum: 2
+      from:
+        orgs:
+          - "zalando"
+# team should be valid team id in team service https://teams.auth.zalando.com/api/teams/:id
+X-Zalando-Team: "acid"
+# type should be one of [code, doc, config, tools, secrets]
+# code will be the default value, if X-Zalando-Type is not found in .zappr.yml
+X-Zalando-Type: code

.zappr.yml

@@ -1,12 +0,0 @@
-approvals:
-  # PR needs at least 4 approvals
-  minimum: 1
-  # approval = comment that matches this regex
-  pattern: "^\\s*(:?\\+1:?|👍)\\s*$"
-  from:
-    # commenter must be either one of:
-    # a public zalando org member
-    orgs:
-      - zalando
-    # a collaborator of the repo
-    collaborators: true

README.rst

@@ -4,7 +4,7 @@ Patroni: A Template for PostgreSQL HA with ZooKeeper, etcd or Consul
 ------------------------------------------------------------
 There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
 
-Patroni is a template for you to create your own customized, high-availability solution using Python and — for maximum accessibility — a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter—or anywhere else—will hopefully find it useful.
+Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
 
 We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely.
 
@@ -58,7 +58,7 @@ To get started, do the following from different terminals:
     > ./patroni.py postgres0.yml
     > ./patroni.py postgres1.yml
 
-You will then see a high-availability cluster start up. Test different settings in the YAML files to see how the cluster’s behavior changes. Kill some of the components to see how the system behaves.
+You will then see a high-availability cluster start up. Test different settings in the YAML files to see how the cluster's behavior changes. Kill some of the components to see how the system behaves.
 
 Add more ``postgres*.yml`` files to create an even larger cluster.
 
@@ -120,7 +120,7 @@ Contributing
 Patroni accepts contributions from the open-source community; see the `Issues Tracker <https://github.com/zalando/patroni/issues>`__ for current needs. 
 
 Before making a contribution, please let us know by posting a comment to the relevant issue. 
-If you would like to propose a new feature, please first file a new issue explaining the feature you’d like to create.
+If you would like to propose a new feature, please first file a new issue explaining the feature you'd like to create.
 
 .. |Build Status| image:: https://travis-ci.org/zalando/patroni.svg?branch=master
    :target: https://travis-ci.org/zalando/patroni

docs/ENVIRONMENT.rst

@@ -38,6 +38,7 @@ PostgreSQL
 -  **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
 -  **PATRONI\_POSTGRESQL\_CONNECT\_ADDRESS**: IP address + port through which Postgres is accessible from other nodes and applications.
 -  **PATRONI\_POSTGRESQL\_DATA\_DIR**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
+-  **PATRONI\_POSTGRESQL\_BIN_DIR**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The  default value is an empty string meaning that PATH environment variable will be used to find the executables.
 -  **PATRONI\_POSTGRESQL\_PGPASS**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni.
 -  **PATRONI\_REPLICATION\_USERNAME**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication
 -  **PATRONI\_REPLICATION\_PASSWORD**: replication password; the user will be created during initialization.

docs/SETTINGS.rst

@@ -65,9 +65,11 @@ PostgreSQL
 -  **connect\_address**: IP address + port through which Postgres is accessible from other nodes and applications.
 -  **create\_replica\_methods**: an ordered list of the create methods for turning a Patroni node into a new replica. "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its own config item.
 -  **data\_dir**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
+-  **bin\_dir**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The  default value is an empty string meaning that PATH environment variable will be used to find the executables.
 -  **listen**: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
 -  **pgpass**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni.
 -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower.
+-  **custom_conf** : path to an optional custom ``postgresql.conf`` file, that will be used in place of ``postgresql.base.conf``. The file must exist on all cluster nodes, be readable by PostgreSQL and will be included from its location on the real ``postgresql.conf``. Note that Patroni will not monitor this file for changes, nor backup it. However, its settings can still be overriden by Patroni's own configuration facilities - see `dynamic configuration <https://github.com/zalando/patroni/blob/master/docs/dynamic_configuration.rst>`__ for details.
 -  **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.
 -  **pg\_ctl\_timeout**: How long should pg_ctl wait when doing ``start``, ``stop`` or ``restart``. Default value is 60 seconds.
 -  **use\_pg\_rewind**: try to use pg\_rewind on the former leader when it joins cluster as a replica.

docs/dynamic_configuration.rst

@@ -48,23 +48,24 @@ To be on the safe side parameters from the above lists are not written into ``po
 
 When applying the local or dynamic configuration options, the following actions are taken:
 
-- The node first checks if there is a postgresql.base.conf.
+- The node first checks if there is a postgresql.base.conf or if the ``custom_conf`` parameter is set.
-- If it exists, it contains the renamed "original" configuration.
+- If the `custom_conf` parameter is set, it will take the file specified on it as a base configuration, ignoring `postgresql.base.conf` and `postgresql.conf`.
-- If it doesn't, the original postgresql.conf is taken and renamed to postgresql.base.conf.
+- If the `custom_conf` parameter is not set and `postgresql.base.conf` exists, it contains the renamed "original" configuration and it will be used as a base configuration.
+- If there is no `custom_conf` nor `postgresql.base.conf`, the original postgresql.conf is taken and renamed to postgresql.base.conf.
 - The dynamic options (with the exceptions above) are dumped into the postgresql.conf and an include is set in
-  postgresql.conf to postgresql.base.conf. Therefore, we would be able to apply new options without re-reading the configuration file to check if the include is present not.
+  postgresql.conf to the used base configuration (either postgresql.base.conf or what is on ``custom_conf``). Therefore, we would be able to apply new options without re-reading the configuration file to check if the include is present not.
 - Some parameters that are essential for Patroni to manage the cluster are overridden using the command line.
 - If some of the options that require restart are changed (we should look at the context in pg_settings and at the actual
   values of those options), a pending_restart flag of a given node is set. This flag is reset on any restart.
 
 The parameters would be applied in the following order (run-time are given the highest priority):
 
-1. load parameters from file `postgresql.base.conf`
+1. load parameters from file `postgresql.base.conf` (or from a `custom_conf` file, if set)
 2. load parameters from file `postgresql.conf`
 3. load parameters from file `postgresql.auto.conf`
 4. run-time parameter using `-o --name=value`
 
-This allows configuration for all the nodes (2), configuration for a specific node using `ALTER SYSTEM` (3) and ensures that parameters essential to the running of Patroni are enforced. (4)
+This allows configuration for all the nodes (2), configuration for a specific node using `ALTER SYSTEM` (3) and ensures that parameters essential to the running of Patroni are enforced (4), as well as leaves room for configuration tools that manage `postgresql.conf` directly without involving Patroni (1).
 
 
 Also, the following Patroni configuration options can be changed only dynamically:

docs/pause.rst

@@ -0,0 +1,33 @@
+Pause/Resume mode for the cluster
+=================================
+
+The goal
+--------
+
+Under certain circumstances Patroni needs to temporary step down from managing the cluster, while still retaining the cluster state in DCS. Possible use cases are uncommon activities on the cluster, such as major version upgrades or corruption recovery. During those activities nodes are often started and stopped for the reason unknown to Patroni, some nodes can be even temporary promoted, violating the assumption of running only one master. Therefore, Patroni needs to be able to "detach" from the running cluster, implementing an equivalent of the maintenance mode in Pacemaker.
+
+
+
+The implementation
+------------------
+
+When Patroni runs in a paused mode, it does not change the state of PostgreSQL, except for the following cases:
+
+- For each node, the member key in DCS is updated with the current information about the cluster. This causes Patroni to run read-only queries on a member node if the member is running.
+
+- For the Postgres master with the leader lock Patroni updates the lock. If the node with the leader lock stops being the master (i.e. is demoted manually), Patroni will release the lock instead of promoting the node back.
+
+- Manual unscheduled restart, reinitialize and manual failover are allowed. Manual failover is only allowed if the node to failover to is specified. In the paused mode, manual failover does not require a running master node.
+
+- If 'parallel' masters are detected by Patroni, it emits a warning, but does not demote the masters without the leader lock.
+
+- If there is no leader lock in the cluster, the running master acquires the lock. If there is more than one master node, then the first master to acquire the lock wins. If there are no masters altogether, Patroni does not try to promote any replicas. There is an exception in this rule: if there is no leader lock because the old master has demoted itself due to the manual promotion, then only the candidate node mentioned in the promotion request may take the leader lock. When the new leader lock is granted (i.e. after promoting a replica manually), Patroni makes sure the replicas that were streaming from the previous leader will switch to the new one.
+
+- When Postgres is stopped, Patroni does not try to start it. When Patroni is stopped, it does not to stop Postgres instance it is managing.
+
+User guide
+----------
+
+``patronictl`` supports ``pause`` and ``resume`` commands.
+
+One can also issue a ``PATCH`` request to the ``{namespace}/{cluster}/config`` key with ``{"pause": true/false/null}``

features/patroni_api.feature

@@ -12,7 +12,7 @@ Scenario: check API requests on a stand-alone server
 	Then I receive a response code 503
 	When I run patronictl.py reinit batman postgres0 --force
 	Then I receive a response returncode 0
-	And I receive a response output "reinitialize failed for member postgres0, status code=503, (I am the leader, can not reinitialize)"
+	And I receive a response output "Failed: reinitialize for member postgres0, status code=503, (I am the leader, can not reinitialize)"
 	When I run patronictl.py failover batman --master postgres0 --force
 	Then I receive a response returncode 1
 	And I receive a response output "Error: No candidates found to failover to"
@@ -34,46 +34,66 @@ Scenario: check local configuration reload
 	Then I receive a response code 202
 
 Scenario: check dynamic configuration change via DCS
-	Given I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "loop_wait": 1, "postgresql": {"parameters": {"max_connections": 101}}}
+	Given I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "loop_wait": 2, "postgresql": {"parameters": {"max_connections": 101}}}
 	Then I receive a response code 200
-	And I receive a response loop_wait 1
+	And I receive a response loop_wait 2
 	And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 11 seconds
 	When I issue a GET request to http://127.0.0.1:8008/config
 	Then I receive a response code 200
-	And I receive a response loop_wait 1
+	And I receive a response loop_wait 2
 	When I issue a GET request to http://127.0.0.1:8008/patroni
 	Then I receive a response code 200
 	And I receive a response tags {'tag': 'new_value'}
 
-Scenario: check API requests for the primary-replica pair
+Scenario: check API requests for the primary-replica pair in the pause mode
-	Given I start postgres1
+	Given I run patronictl.py pause batman
-	And replication works from postgres0 to postgres1 after 20 seconds
+	Then I receive a response returncode 0
+	When I start postgres1
+	Then replication works from postgres0 to postgres1 after 20 seconds
 	When I issue a GET request to http://127.0.0.1:8009/replica
 	Then I receive a response code 200
 	And I receive a response state running
 	And I receive a response role replica
 	When I run patronictl.py reinit batman postgres1 --force
 	Then I receive a response returncode 0
-	And I receive a response output "Succesful reinitialize on member postgres1"
+	And I receive a response output "Success: reinitialize for member postgres1"
 	When I run patronictl.py restart batman postgres0 --force
 	Then I receive a response returncode 0
-	And I receive a response output "Succesful restart on member postgres0"
+	And I receive a response output "Success: restart on member postgres0"
 	And postgres0 role is the primary after 5 seconds
 	When I sleep for 10 seconds
 	Then postgres1 role is the secondary after 15 seconds
 
-Scenario: check the failover via the API
+Scenario: check the failover via the API in the pause mode
 	Given I run patronictl.py failover batman --master postgres0 --candidate postgres1 --force
 	Then I receive a response returncode 0
 	And postgres1 is a leader after 5 seconds
-	And postgres1 role is the primary after 5 seconds
+	And postgres1 role is the primary after 10 seconds
 	And postgres0 role is the secondary after 10 seconds
 	And replication works from postgres1 to postgres0 after 20 seconds
 
 Scenario: check the scheduled failover
+	Given I issue a scheduled failover from postgres1 to postgres0 in 1 seconds
+	Then I receive a response returncode 1
+	And I receive a response output "Can't schedule failover in the paused state"
+	When I run patronictl.py resume batman
+	Then I receive a response returncode 0
 	Given I issue a scheduled failover from postgres1 to postgres0 in 1 seconds
 	Then I receive a response returncode 0
 	And postgres0 is a leader after 20 seconds
-	And postgres0 role is the primary after 5 seconds
+	And postgres0 role is the primary after 10 seconds
 	And postgres1 role is the secondary after 10 seconds
 	And replication works from postgres0 to postgres1 after 25 seconds
+
+Scenario: check the scheduled restart
+	Given I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"superuser_reserved_connections": "6"}}}
+	Then I receive a response code 200
+		And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 5 seconds
+	Given I issue a scheduled restart at http://127.0.0.1:8008 in 1 seconds with {"role": "replica"}
+	Then I receive a response code 202
+		And I sleep for 2 seconds
+		And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 10 seconds
+	Given I issue a scheduled restart at http://127.0.0.1:8008 in 1 seconds with {"restart_pending": "True"}
+	Then I receive a response code 202
+		And Response on GET http://127.0.0.1:8008/patroni does not contain pending_restart after 10 seconds
+

features/steps/patroni_api.py

@@ -121,18 +121,30 @@ def scheduled_failover(context, from_host, to_host, in_seconds):
     """.format(from_host, to_host, datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds))))
 
 
+@step('I issue a scheduled restart at {url:url} in {in_seconds:d} seconds with {data}')
+def scheduled_restart(context, url, in_seconds, data):
+    data = data and json.loads(data) or {}
+    data.update(schedule='{0}'.format((datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds))).isoformat()))
+    context.execute_steps(u"""Given I issue a POST request to {0}/restart with {1}""".format(url, json.dumps(data)))
+
+
 @step('I add tag {tag:w} {value:w} to {pg_name:w} config')
 def add_tag_to_config(context, tag, value, pg_name):
     context.pctl.add_tag_to_config(pg_name, tag, value)
 
 
 @then('Response on GET {url} contains {value} after {timeout:d} seconds')
-def check_http_response(context, url, value, timeout):
+def check_http_response(context, url, value, timeout, negate=False):
     for _ in range(int(timeout)):
         r = requests.get(url)
-        if value in r.content.decode('utf-8'):
+        if (value in r.content.decode('utf-8')) != negate:
             break
         time.sleep(1)
     else:
         assert False,\
-            "Value {0} is not present in response after {1} seconds".format(value, timeout)
+            "Value {0} is {1} present in response after {2} seconds".format(value, "not" if not negate else "", timeout)
+
+
+@then('Response on GET {url} does not contain {value} after {timeout:d} seconds')
+def check_not_in_http_response(context, url, value, timeout):
+    check_http_response(context, url, value, timeout, negate=True)

mkbinary.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -e
+
+pip install --ignore-installed setuptools==19.2 pyinstaller
+pyinstaller --clean --onefile patroni.spec

patroni.spec

@@ -0,0 +1,39 @@
+# -*- mode: python -*-
+
+block_cipher = None
+
+
+def hiddenimports():
+    import sys
+    sys.path.insert(0, '.')
+    try:
+        import patroni.dcs
+        return patroni.dcs.dcs_modules()
+    finally:
+        sys.path.pop(0)
+
+
+a = Analysis(['patroni/__main__.py'],
+             pathex=[],
+             binaries=None,
+             datas=None,
+             hiddenimports=hiddenimports(),
+             hookspath=[],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher)
+
+pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
+
+exe = EXE(pyz,
+    a.scripts,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    name='patroni',
+    debug=False,
+    strip=False,
+    upx=True,
+    console=True)

patroni/__init__.py

@@ -30,17 +30,19 @@ class Patroni(object):
         self.ha = Ha(self)
 
         self.tags = self.get_tags()
-        self.nap_time = self.config['loop_wait']
         self.next_run = time.time()
+        self.scheduled_restart = {}
 
     def load_dynamic_configuration(self):
         while True:
             try:
                 cluster = self.dcs.get_cluster()
                 if cluster and cluster.config:
-                    self.config.set_dynamic_configuration(cluster.config)
+                    if self.config.set_dynamic_configuration(cluster.config):
+                        self.dcs.reload_config(self.config)
                 elif not self.config.dynamic_configuration and 'bootstrap' in self.config:
-                    self.config.set_dynamic_configuration(self.config['bootstrap']['dcs'])
+                    if self.config.set_dynamic_configuration(self.config['bootstrap']['dcs']):
+                        self.dcs.reload_config(self.config)
                 break
             except DCSError:
                 logger.warning('Can not get cluster from dcs')
@@ -51,14 +53,12 @@ class Patroni(object):
 
     @property
     def nofailover(self):
-        return self.tags.get('nofailover', False)
+        return bool(self.tags.get('nofailover', False))
 
     def reload_config(self):
         try:
             self.tags = self.get_tags()
-            self.nap_time = self.config['loop_wait']
+            self.dcs.reload_config(self.config)
-            self.dcs.set_ttl(self.config.get('ttl') or 30)
-            self.dcs.set_retry_timeout(self.config.get('retry_timeout') or self.nap_time)
             self.api.reload_config(self.config['restapi'])
             self.postgresql.reload_config(self.config['postgresql'])
         except Exception:
@@ -78,14 +78,18 @@ class Patroni(object):
 
     @property
     def noloadbalance(self):
-        return self.tags.get('noloadbalance', False)
+        return bool(self.tags.get('noloadbalance', False))
 
     def schedule_next_run(self):
-        self.next_run += self.nap_time
+        self.next_run += self.dcs.loop_wait
         current_time = time.time()
         nap_time = self.next_run - current_time
         if nap_time <= 0:
             self.next_run = current_time
+            # Release the GIL so we don't starve anyone waiting on async_executor lock
+            time.sleep(0.001)
+            # Warn user that Patroni is not keeping up
+            logger.warning("Loop time exceeded, rescheduling immediately.")
         elif self.dcs.watch(nap_time):
             self.next_run = time.time()
 
@@ -130,5 +134,8 @@ def main():
         pass
     finally:
         patroni.api.shutdown()
-        patroni.postgresql.stop(checkpoint=False)
+        if patroni.ha.is_paused():
-        patroni.dcs.delete_leader()
+            logger.info('Leader key is not deleted and Postgresql is not stopped due paused state')
+        else:
+            patroni.postgresql.stop(checkpoint=False)
+            patroni.dcs.delete_leader()

patroni/api.py

@@ -9,7 +9,7 @@ import datetime
 import pytz
 
 from patroni.exceptions import PostgresConnectionException
-from patroni.utils import deep_compare, patch_config, Retry, RetryFailedError
+from patroni.utils import deep_compare, patch_config, Retry, RetryFailedError, is_valid_pg_version
 from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 from six.moves.socketserver import ThreadingMixIn
 from threading import Thread
@@ -63,6 +63,10 @@ class RestApiHandler(BaseHTTPRequestHandler):
         if patroni.postgresql.pending_restart:
             response['pending_restart'] = True
         response['patroni'] = {'version': patroni.version, 'scope': patroni.postgresql.scope}
+        if patroni.scheduled_restart and isinstance(patroni.scheduled_restart, dict):
+            response['scheduled_restart'] = patroni.scheduled_restart.copy()
+            del response['scheduled_restart']['postmaster_start_time']
+            response['scheduled_restart']['schedule'] = (response['scheduled_restart']['schedule']).isoformat()
         self._write_json_response(status_code, response)
 
     def do_GET(self, write_status_code_only=False):
@@ -106,19 +110,21 @@ class RestApiHandler(BaseHTTPRequestHandler):
         self._write_status_response(200, response)
 
     def do_GET_config(self):
-        cluster = self.server.patroni.ha.dcs.cluster or self.server.patroni.ha.dcs.get_cluster()
+        cluster = self.server.patroni.dcs.cluster or self.server.patroni.dcs.get_cluster()
         if cluster.config:
             self._write_json_response(200, cluster.config.data)
         else:
             self.send_error(502)
 
-    def _read_json_content(self):
+    def _read_json_content(self, body_is_optional=False):
         if 'content-length' not in self.headers:
-            return self.send_error(411)
+            return self.send_error(411) if not body_is_optional else {}
         try:
             content_length = int(self.headers.get('content-length'))
+            if content_length == 0 and body_is_optional:
+                return {}
             request = json.loads(self.rfile.read(content_length).decode('utf-8'))
-            if isinstance(request, dict) and request:
+            if isinstance(request, dict) and (request or body_is_optional):
                 return request
         except Exception:
             logger.exception('Bad request')
@@ -128,11 +134,11 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_PATCH_config(self):
         request = self._read_json_content()
         if request:
-            cluster = self.server.patroni.ha.dcs.get_cluster()
+            cluster = self.server.patroni.dcs.get_cluster()
             data = cluster.config.data.copy()
             if patch_config(data, request):
                 value = json.dumps(data, separators=(',', ':'))
-                if not self.server.patroni.ha.dcs.set_config_value(value, cluster.config.index):
+                if not self.server.patroni.dcs.set_config_value(value, cluster.config.index):
                     return self.send_error(409)
             self._write_json_response(200, data)
 
@@ -140,10 +146,10 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_PUT_config(self):
         request = self._read_json_content()
         if request:
-            cluster = self.server.patroni.ha.dcs.get_cluster()
+            cluster = self.server.patroni.dcs.get_cluster()
             if not deep_compare(request, cluster.config.data):
                 value = json.dumps(request, separators=(',', ':'))
-                if not self.server.patroni.ha.dcs.set_config_value(value):
+                if not self.server.patroni.dcs.set_config_value(value):
                     return self.send_error(502)
             self._write_json_response(200, request)
 
@@ -162,39 +168,103 @@ class RestApiHandler(BaseHTTPRequestHandler):
             response = str(e)
         self._write_response(status_code, response)
 
+    @staticmethod
+    def parse_schedule(schedule, action):
+        """ parses the given schedule and validates at """
+        error = None
+        scheduled_at = None
+        try:
+            scheduled_at = dateutil.parser.parse(schedule)
+            if scheduled_at.tzinfo is None:
+                error = 'Timezone information is mandatory for the scheduled {0}'.format(action)
+                status_code = 400
+            elif scheduled_at < datetime.datetime.now(pytz.utc):
+                error = 'Cannot schedule {0} in the past'.format(action)
+                status_code = 422
+            else:
+                status_code = None
+        except (ValueError, TypeError):
+            logger.exception('Invalid scheduled %s time: %s', action, schedule)
+            error = 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601'
+            status_code = 422
+        return (status_code, error, scheduled_at)
+
     @check_auth
     def do_POST_restart(self):
         status_code = 500
         data = 'restart failed'
-        try:
+        request = self._read_json_content(body_is_optional=True)
-            status, data = self.server.patroni.ha.restart()
+        cluster = self.server.patroni.dcs.get_cluster()
-            status_code = 200 if status else 503
+        if request is None:
-        except Exception:
+            # failed to parse the json
-            logger.exception('Exception during restart')
+            return
+        if request:
+            logger.debug("received restart request: {0}".format(request))
+
+        if cluster.is_paused() and 'schedule' in request:
+            self._write_response(status_code, "Can't schedule restart in the paused state")
+            return
+
+        for k in request:
+            if k == 'schedule':
+                (_, data, request[k]) = self.parse_schedule(request[k], "restart")
+                if _:
+                    status_code = _
+                    break
+            elif k == 'role':
+                if request[k] not in ('master', 'replica'):
+                    status_code = 400
+                    data = "PostgreSQL role should be either master or replica"
+                    break
+            elif k == 'postgres_version':
+                if not is_valid_pg_version(request[k]):
+                    status_code = 400
+                    data = "PostgreSQL version should be in the first.major.minor format"
+                    break
+            elif k != 'restart_pending':
+                status_code = 400
+                data = "Unknown filter for the scheduled restart: {0}".format(k)
+                break
+        else:
+            if 'schedule' not in request:
+                try:
+                    status, data = self.server.patroni.ha.restart(request)
+                    status_code = 200 if status else 503
+                except Exception:
+                    logger.exception('Exception during restart')
+                    status_code = 400
+            else:
+                request['postmaster_start_time'] = self.server.patroni.ha.state_handler.postmaster_start_time()
+                if self.server.patroni.ha.schedule_future_restart(request):
+                    data = "Restart scheduled"
+                    status_code = 202
+                else:
+                    data = "Another restart is already scheduled"
+                    status_code = 409
         self._write_response(status_code, data)
 
     @check_auth
-    def do_POST_reinitialize(self):
+    def do_DELETE_restart(self):
-        ha = self.server.patroni.ha
+        if self.server.patroni.ha.delete_future_restart():
-        cluster = ha.dcs.get_cluster()
+            data = "scheduled restart deleted"
-        if cluster.is_unlocked():
+            code = 200
-            status_code = 503
-            data = 'Cluster has no leader, can not reinitialize'
-        elif cluster.leader.name == ha.state_handler.name:
-            status_code = 503
-            data = 'I am the leader, can not reinitialize'
         else:
-            action = ha.schedule_reinitialize()
+            data = "no restarts are scheduled"
-            if action is not None:
+            code = 404
-                status_code = 503
+        self._write_response(code, data)
-                data = action + ' already in progress'
+
-            else:
+    @check_auth
-                status_code = 200
+    def do_POST_reinitialize(self):
-                data = 'reinitialize scheduled'
+        data = self.server.patroni.ha.reinitialize()
+        if data is None:
+            status_code = 200
+            data = 'reinitialize started'
+        else:
+            status_code = 503
         self._write_response(status_code, data)
 
     def poll_failover_result(self, leader, candidate):
-        timeout = 10 if self.server.patroni.nap_time < 10 else self.server.patroni.nap_time
+        timeout = max(10, self.server.patroni.dcs.loop_wait)
         for _ in range(0, timeout*2):
             time.sleep(1)
             try:
@@ -211,7 +281,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         return 503, 'Failover status unknown'
 
     def is_failover_possible(self, cluster, leader, candidate):
-        if leader and not cluster.leader or cluster.leader.name != leader:
+        if leader and (not cluster.leader or cluster.leader.name != leader):
             return 'leader name does not match'
         if candidate:
             members = [m for m in cluster.members if m.name == candidate]
@@ -229,14 +299,17 @@ class RestApiHandler(BaseHTTPRequestHandler):
     @check_auth
     def do_POST_failover(self):
         request = self._read_json_content()
+        status_code = 500
         if not request:
             return
 
         leader = request.get('leader')
         candidate = request.get('candidate') or request.get('member')
         scheduled_at = request.get('scheduled_at')
-        cluster = self.server.patroni.ha.dcs.get_cluster()
+        cluster = self.server.patroni.dcs.get_cluster()
-        status_code = 500
+
+        if scheduled_at and cluster.is_paused():
+            self._write_response(status_code, "Can't schedule failover in the paused state")
 
         logger.info("received failover request with leader=%s candidate=%s scheduled_at=%s",
                     leader, candidate, scheduled_at)
@@ -244,25 +317,16 @@ class RestApiHandler(BaseHTTPRequestHandler):
         data = ''
         if leader or candidate:
             if scheduled_at:
-                try:
+                (_, data, scheduled_at) = self.parse_schedule(scheduled_at, "failover")
-                    scheduled_at = dateutil.parser.parse(scheduled_at)
+                if _:
-                    if scheduled_at.tzinfo is None:
+                    status_code = _
-                        data = 'Timezone information is mandatory for scheduled_at'
+                elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
-                        status_code = 400
+                    self.server.patroni.dcs.event.set()
-                    elif scheduled_at < datetime.datetime.now(pytz.utc):
+                    data = 'Failover scheduled'
-                        data = 'Cannot schedule failover in the past'
+                    status_code = 202
-                        status_code = 422
+                else:
-                    elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
+                    data = 'failed to write failover key into DCS'
-                        self.server.patroni.dcs.event.set()
+                    status_code = 503
-                        data = 'Failover scheduled'
-                        status_code = 202
-                    else:
-                        data = 'failed to write failover key into DCS'
-                        status_code = 503
-                except (ValueError, TypeError):
-                    logger.exception('Invalid scheduled failover time: %s', request['scheduled_at'])
-                    data = 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601'
-                    status_code = 422
             else:
                 data = self.is_failover_possible(cluster, leader, candidate)
                 if not data:
@@ -303,7 +367,11 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
     def get_postgresql_status(self, retry=False):
         try:
-            row = self.query("""SELECT to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
+            row = self.query("""WITH replication_info AS (
+                                    SELECT usename, application_name, client_addr, state, sync_state, sync_priority
+                                      FROM pg_stat_replication
+                                )
+                                SELECT to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
                                        pg_is_in_recovery(),
                                        CASE WHEN pg_is_in_recovery()
                                             THEN 0
@@ -312,8 +380,10 @@ class RestApiHandler(BaseHTTPRequestHandler):
                                        pg_xlog_location_diff(pg_last_xlog_receive_location(), '0/0')::bigint,
                                        pg_xlog_location_diff(pg_last_xlog_replay_location(), '0/0')::bigint,
                                        to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
-                                       pg_is_in_recovery() AND pg_is_xlog_replay_paused()""", retry=retry)[0]
+                                       pg_is_in_recovery() AND pg_is_xlog_replay_paused(),
-            return {
+                                       (SELECT json_agg(row_to_json(ri)) FROM replication_info ri)""", retry=retry)[0]
+
+            result = {
                 'state': self.server.patroni.postgresql.state,
                 'postmaster_start_time': row[0],
                 'role': 'replica' if row[1] else 'master',
@@ -326,6 +396,11 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     'location': row[2]
                 })
             }
+
+            if row[7]:
+                result['replication'] = row[7]
+
+            return result
         except (psycopg2.Error, RetryFailedError, PostgresConnectionException):
             state = self.server.patroni.postgresql.state
             if state == 'running':

patroni/async_executor.py

@@ -1,5 +1,5 @@
 import logging
-from threading import Lock, Thread
+from threading import RLock, Thread
 
 logger = logging.getLogger(__name__)
 
@@ -7,21 +7,19 @@ logger = logging.getLogger(__name__)
 class AsyncExecutor(object):
 
     def __init__(self):
-        self._busy = False
+        self._thread_lock = RLock()
-        self._thread_lock = Lock()
         self._scheduled_action = None
-        self._scheduled_action_lock = Lock()
+        self._scheduled_action_lock = RLock()
 
     @property
     def busy(self):
-        return self._busy
+        return self.scheduled_action is not None
 
     def schedule(self, action, immediately=False):
         with self._scheduled_action_lock:
             if self._scheduled_action is not None:
                 return self._scheduled_action
             self._scheduled_action = action
-            self._busy = immediately
         return None
 
     @property
@@ -40,11 +38,9 @@ class AsyncExecutor(object):
             logger.exception('Exception during execution of long running task %s', self.scheduled_action)
         finally:
             with self:
-                self._busy = False
                 self.reset_scheduled_action()
 
     def run_async(self, func, args=()):
-        self._busy = True
         Thread(target=self.run, args=(func, args)).start()
 
     def __enter__(self):

patroni/config.py

@@ -42,6 +42,7 @@ class Config(object):
         'ttl': 30, 'loop_wait': 10, 'retry_timeout': 10,
         'maximum_lag_on_failover': 1048576,
         'postgresql': {
+            'bin_dir': '',
             'use_slots': True,
             'parameters': {p: v[0] for p, v in Postgresql.CMDLINE_OPTIONS.items()}
         }
@@ -64,7 +65,7 @@ class Config(object):
                 print('Usage: {0} config.yml'.format(sys.argv[0]))
                 print('\tPatroni may also read the configuration from the {0} environment variable'.
                       format(self.PATRONI_CONFIG_VARIABLE))
-                exit(1)
+                sys.exit(1)
 
         self.__effective_configuration = self._build_effective_configuration({}, self._local_configuration)
         self._data_dir = self.__effective_configuration['postgresql']['data_dir']
@@ -193,7 +194,7 @@ class Config(object):
                     ret[section][param] = value
 
         _set_section_values('restapi', ['listen', 'connect_address', 'certfile', 'keyfile'])
-        _set_section_values('postgresql', ['listen', 'connect_address', 'data_dir', 'pgpass'])
+        _set_section_values('postgresql', ['listen', 'connect_address', 'data_dir', 'pgpass', 'bin_dir'])
 
         def _get_auth(name):
             ret = {}

patroni/ctl.py

@@ -21,7 +21,8 @@ from click import ClickException
 from patroni.config import Config
 from patroni.dcs import get_dcs as _get_dcs
 from patroni.exceptions import PatroniException
-from patroni.postgresql import get_conn_kwargs
+from patroni.postgresql import Postgresql
+from patroni.utils import is_valid_pg_version
 from prettytable import PrettyTable
 from six.moves.urllib_parse import urlparse
 
@@ -76,7 +77,6 @@ def load_config(path, dcs):
         for d in DCS_DEFAULTS:
             config.pop(d, None)
         config.update(dcs)
-
     return config
 
 
@@ -106,7 +106,7 @@ def ctl(ctx):
 
 
 def get_dcs(config, scope):
-    config.setdefault('scope', scope)
+    config['scope'] = scope
     config.setdefault('name', scope)
     try:
         return _get_dcs(config)
@@ -119,15 +119,17 @@ def auth_header(config):
         return {'Authorization': 'Basic ' + base64.b64encode(config['restapi']['auth'].encode('utf-8')).decode('utf-8')}
 
 
-def post_patroni(member, endpoint, content, headers=None):
+def request_patroni(member, request_type, endpoint, content=None, headers=None):
     headers = headers or {}
-    url = urlparse(member.api_url)
+    url_parts = urlparse(member.api_url)
-    logging.debug(url)
+    logging.debug(url_parts)
     if 'Content-Type' not in headers:
         headers['Content-Type'] = 'application/json'
-    return requests.post('{0}://{1}/{2}'.format(url.scheme, url.netloc, endpoint),
+
-                         headers=headers,
+    url = '{0}://{1}/{2}'.format(url_parts.scheme, url_parts.netloc, endpoint)
-                         data=json.dumps(content), timeout=60)
+
+    return getattr(requests, request_type)(url, headers=headers,
+                                           data=json.dumps(content) if content else None, timeout=60)
 
 
 def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True, delimiter='\t'):
@@ -182,16 +184,6 @@ def watching(w, watch, max_count=None, clear=True):
         yield 0
 
 
-def build_connect_parameters(conn_url, connect_parameters):
-    params = get_conn_kwargs(conn_url, connect_parameters)
-    params.update({'fallback_application_name': 'Patroni ctl', 'connect_timeout': '5'})
-    if 'database' in connect_parameters:
-        params['database'] = connect_parameters['database']
-    else:
-        params.pop('database')
-    return params
-
-
 def get_all_members(cluster, role='master'):
     if role == 'master':
         if cluster.leader is not None:
@@ -216,7 +208,12 @@ def get_cursor(cluster, connect_parameters, role='master', member=None):
     if member is None:
         return None
 
-    params = build_connect_parameters(member.conn_url, connect_parameters)
+    params = member.conn_kwargs(connect_parameters)
+    params.update({'fallback_application_name': 'Patroni ctl', 'connect_timeout': '5'})
+    if 'database' in connect_parameters:
+        params['database'] = connect_parameters['database']
+    else:
+        params.pop('database')
 
     conn = psycopg2.connect(**params)
     conn.autocommit = True
@@ -235,6 +232,37 @@ def get_cursor(cluster, connect_parameters, role='master', member=None):
     return None
 
 
+def get_members(cluster, cluster_name, member_names, role, force, action):
+    candidates = {m.name: m for m in cluster.members}
+
+    if not force or role:
+        output_members(cluster, cluster_name)
+
+    if role:
+        role_names = [m.name for m in get_all_members(cluster, role)]
+        if member_names:
+            member_names = list(set(member_names) & set(role_names))
+            if not member_names:
+                raise PatroniCtlException('No {0} among provided members'.format(role))
+        else:
+            member_names = role_names
+
+    if not member_names and not force:
+        member_names = [click.prompt('Which member do you want to {0} [{1}]?'.format(action,
+                        ', '.join(candidates.keys())), type=str, default='')]
+
+    for mn in member_names:
+        if mn not in candidates:
+            raise PatroniCtlException('{0} is not a member of cluster'.format(mn))
+
+    if not force:
+        confirm = click.confirm('Are you sure you want to {0} members {1}?'.format(action, ', '.join(member_names)))
+        if not confirm:
+            raise PatroniCtlException('Aborted {0}'.format(action))
+
+    return [candidates[n] for n in member_names]
+
+
 @ctl.command('dsn', help='Generate a dsn for the provided member, defaults to a dsn of the master')
 @click.option('--role', '-r', help='Give a dsn of any member with this role', type=click.Choice(['master', 'replica',
               'any']), default=None)
@@ -253,7 +281,7 @@ def dsn(cluster_name, config_file, dcs, role, member):
     if m is None:
         raise PatroniCtlException('Can not find a suitable member')
 
-    params = get_conn_kwargs(m.conn_url)
+    params = m.conn_kwargs()
     click.echo('host={host} port={port}'.format(**params))
 
 
@@ -363,7 +391,7 @@ def query_member(cluster, cursor, member, role, command, connect_parameters):
 def remove(config_file, cluster_name, fmt, dcs):
     _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
 
-    output_members(cluster, cluster_name, fmt)
+    output_members(cluster, cluster_name, fmt=fmt)
 
     confirm = click.prompt('Please confirm the cluster name to remove', type=str)
     if confirm != cluster_name:
@@ -398,30 +426,6 @@ def wait_for_leader(dcs, timeout=30):
     raise PatroniCtlException('Timeout occured')
 
 
-def empty_post_to_members(cluster, member_names, force, endpoint, headers=None):
-    candidates = {m.name: m for m in cluster.members}
-
-    if not member_names:
-        member_names = [click.prompt('Which member do you want to {0} [{1}]?'.format(endpoint,
-                        ', '.join(candidates.keys())), type=str, default='')]
-
-    for mn in member_names:
-        if mn not in candidates:
-            raise PatroniCtlException('{0} is not a member of cluster'.format(mn))
-
-    if not force:
-        confirm = click.confirm('Are you sure you want to {0} members {1}?'.format(endpoint, ', '.join(member_names)))
-        if not confirm:
-            raise PatroniCtlException('Aborted {0}'.format(endpoint))
-
-    for mn in member_names:
-        r = post_patroni(candidates[mn], endpoint, '', headers)
-        if r.status_code != 200:
-            click.echo('{0} failed for member {1}, status code={2}, ({3})'.format(endpoint, mn, r.status_code, r.text))
-        else:
-            click.echo('Succesful {0} on member {1}'.format(endpoint, mn))
-
-
 def ctl_load_config(cluster_name, config_file, dcs):
     config = load_config(config_file, dcs)
     dcs = get_dcs(config, cluster_name)
@@ -430,31 +434,92 @@ def ctl_load_config(cluster_name, config_file, dcs):
     return config, dcs, cluster
 
 
+def check_response(response, member_name, action_name, silent_success=False):
+    if response.status_code >= 400:
+        click.echo('Failed: {0} for member {1}, status code={2}, ({3})'.format(
+            action_name, member_name, response.status_code, response.text
+        ))
+    elif not silent_success:
+        click.echo('Success: {0} for member {1}'.format(action_name, member_name))
+
+
+def parse_scheduled(scheduled):
+    if (scheduled or 'now') != 'now':
+        try:
+            scheduled_at = dateutil.parser.parse(scheduled)
+            if scheduled_at.tzinfo is None:
+                scheduled_at = tzlocal.get_localzone().localize(scheduled_at)
+        except (ValueError, TypeError):
+            message = 'Unable to parse scheduled timestamp ({0}). It should be in an unambiguous format (e.g. ISO 8601)'
+            raise PatroniCtlException(message.format(scheduled))
+        return scheduled_at
+
+    return None
+
+
 @ctl.command('restart', help='Restart cluster member')
 @click.argument('cluster_name')
 @click.argument('member_names', nargs=-1)
 @click.option('--role', '-r', help='Restart only members with this role', default='any',
               type=click.Choice(['master', 'replica', 'any']))
 @click.option('--any', 'p_any', help='Restart a single member only', is_flag=True)
+@click.option('--scheduled', help='Timestamp of a scheduled restart in unambiguous format (e.g. ISO 8601)',
+              default=None)
+@click.option('--pg-version', 'version', help='Restart if the PostgreSQL version is less than provided (e.g. 9.5.2)',
+              default=None)
+@click.option('--pending', help='Restart if pending', is_flag=True)
 @option_config_file
 @option_force
 @option_dcs
-def restart(cluster_name, member_names, config_file, dcs, force, role, p_any):
+def restart(cluster_name, member_names, config_file, dcs, force, role, p_any, scheduled, version, pending):
     config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
 
-    role_names = [m.name for m in get_all_members(cluster, role)]
+    members = get_members(cluster, cluster_name, member_names, role, force, 'restart')
-
-    if member_names:
-        member_names = list(set(member_names) & set(role_names))
-    else:
-        member_names = role_names
-
     if p_any:
-        random.shuffle(member_names)
+        random.shuffle(members)
-        member_names = member_names[:1]
+        members = members[:1]
 
-    output_members(cluster, cluster_name)
+    if version is None and not force:
-    empty_post_to_members(cluster, member_names, force, 'restart', auth_header(config))
+        version = click.prompt('Restart if the PostgreSQL version is less than provided (e.g. 9.5.2) ',
+                               type=str, default='')
+
+    content = {}
+    if pending:
+        content['restart_pending'] = True
+
+    if version:
+        if not is_valid_pg_version(version):
+            message = 'PostgreSQL version should be in the first.major.minor format'
+            raise PatroniCtlException(message)
+        else:
+            content['postgres_version'] = version
+
+    if scheduled is None and not force:
+        scheduled = click.prompt('When should the restart take place (e.g. 2015-10-01T14:30) ', type=str, default='now')
+
+    scheduled_at = parse_scheduled(scheduled)
+    if scheduled_at:
+        if cluster.is_paused():
+            raise PatroniCtlException("Can't schedule restart in the paused state")
+        content['schedule'] = scheduled_at.isoformat()
+
+    for member in members:
+        if 'schedule' in content:
+            if force and member.data.get('scheduled_restart'):
+                r = request_patroni(member, 'delete', 'restart', headers=auth_header(config))
+                check_response(r, member.name, 'flush scheduled restart', True)
+
+        r = request_patroni(member, 'post', 'restart', content, auth_header(config))
+        if r.status_code == 200:
+            click.echo('Success: restart on member {0}'.format(member.name))
+        elif r.status_code == 202:
+            click.echo('Success: restart scheduled on member {0}'.format(member.name))
+        elif r.status_code == 409:
+            click.echo('Failed: another restart is already scheduled on member {0}'.format(member.name))
+        else:
+            click.echo('Failed: restart for member {0}, status code={1}, ({2})'.format(
+                member.name, r.status_code, r.text)
+            )
 
 
 @ctl.command('reinit', help='Reinitialize cluster member')
@@ -465,7 +530,11 @@ def restart(cluster_name, member_names, config_file, dcs, force, role, p_any):
 @option_dcs
 def reinit(cluster_name, member_names, config_file, dcs, force):
     config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
-    empty_post_to_members(cluster, member_names, force, 'reinitialize', auth_header(config))
+    members = get_members(cluster, cluster_name, member_names, None, force, 'reinitialize')
+
+    for member in members:
+        r = request_patroni(member, 'post', 'reinitialize', headers=auth_header(config))
+        check_response(r, member.name, 'reinitialize')
 
 
 @ctl.command('failover', help='Failover to a replica')
@@ -474,7 +543,7 @@ def reinit(cluster_name, member_names, config_file, dcs, force):
 @click.option('--candidate', help='The name of the candidate', default=None)
 @click.option('--scheduled', help='Timestamp of a scheduled failover in unambiguous format (e.g. ISO 8601)',
               default=None)
-@click.option('--force', is_flag=True)
+@option_force
 @option_config_file
 @option_dcs
 def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled):
@@ -487,16 +556,16 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
 
     config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
 
-    if cluster.leader is None:
+    if cluster.leader is None and not cluster.is_paused():
         raise PatroniCtlException('This cluster has no master')
 
-    if master is None:
+    if master is None and (not cluster.is_paused() or cluster.leader):
         if force:
             master = cluster.leader.member.name
         else:
             master = click.prompt('Master', type=str, default=cluster.leader.member.name)
 
-    if cluster.leader.member.name != master:
+    if master is not None and cluster.leader and cluster.leader.member.name != master:
         raise PatroniCtlException('Member {0} is not the leader of cluster {1}'.format(master, cluster_name))
 
     candidate_names = [str(m.name) for m in cluster.members if m.name != master]
@@ -519,19 +588,15 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
         scheduled = click.prompt('When should the failover take place (e.g. 2015-10-01T14:30) ', type=str,
                                  default='now')
 
-    if (scheduled or 'now') == 'now':
+    scheduled_at = parse_scheduled(scheduled)
-        scheduled_at = None
+
-    else:
+    if scheduled_at:
-        try:
+        if cluster.is_paused():
-            scheduled_at = dateutil.parser.parse(scheduled)
+            raise PatroniCtlException("Can't schedule failover in the paused state")
-            if scheduled_at.tzinfo is None:
-                scheduled_at = tzlocal.get_localzone().localize(scheduled_at)
-        except (ValueError, TypeError):
-            message = 'Unable to parse scheduled timestamp ({0}). It should be in an unambiguous format (e.g. ISO 8601)'
-            raise PatroniCtlException(message.format(scheduled))
         scheduled_at = scheduled_at.isoformat()
 
     failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at}
+
     logging.debug(failover_value)
 
     # By now we have established that the leader exists and the candidate exists
@@ -547,7 +612,9 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
 
     r = None
     try:
-        r = post_patroni(cluster.leader.member, 'failover', failover_value, auth_header(config))
+        member = cluster.leader.member if cluster.leader else [m for m in cluster.members if m.name == candidate][0]
+
+        r = request_patroni(member, 'post', 'failover', failover_value, auth_header(config))
         if r.status_code in (200, 202):
             logging.debug(r)
             cluster = dcs.get_cluster()
@@ -566,7 +633,7 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
     output_members(cluster, cluster_name)
 
 
-def output_members(cluster, name, fmt='pretty'):
+def output_members(cluster, name, extended=False, fmt='pretty'):
     rows = []
     logging.debug(cluster)
     leader_name = None
@@ -584,21 +651,32 @@ def output_members(cluster, name, fmt='pretty'):
         if m.name == leader_name:
             leader = '*'
 
-        host = get_conn_kwargs(m.conn_url)['host']
+        host = m.conn_kwargs()['host']
 
         xlog_location = m.data.get('xlog_location') or 0
         lag = ''
-        if (xlog_location_cluster >= xlog_location):
+        if xlog_location_cluster >= xlog_location:
             lag = round((xlog_location_cluster - xlog_location)/1024/1024)
 
-        rows.append([
+        row = [
             name,
             m.name,
             host,
             leader,
             m.data.get('state', ''),
-            lag
+            lag,
-        ])
+        ]
+        if extended:
+            value = ''
+            scheduled_restart = m.data.get('scheduled_restart')
+            if scheduled_restart:
+                value = scheduled_restart['schedule']
+                if 'postgres_version' in scheduled_restart:
+                    value += ' if version < {0}'.format(scheduled_restart['postgres_version'])
+
+            row.append(value)
+
+        rows.append(row)
 
     columns = [
         'Cluster',
@@ -610,17 +688,22 @@ def output_members(cluster, name, fmt='pretty'):
     ]
     alignment = {'Cluster': 'l', 'Member': 'l', 'Host': 'l', 'Lag in MB': 'r'}
 
+    if extended:
+        columns.append('Scheduled restart')
+        alignment['Scheduled restart'] = 'l'
+
     print_output(columns, rows, alignment, fmt)
 
 
 @ctl.command('list', help='List the Patroni members for a given Patroni')
 @click.argument('cluster_names', nargs=-1)
+@click.option('--extended', '-e', help='Show some extra information', is_flag=True)
 @option_config_file
 @option_format
 @option_watch
 @option_watchrefresh
 @option_dcs
-def members(config_file, cluster_names, fmt, watch, w, dcs):
+def members(config_file, cluster_names, fmt, watch, w, dcs, extended):
     if not cluster_names:
         logging.warning('Listing members: No cluster names were provided')
         return
@@ -630,7 +713,8 @@ def members(config_file, cluster_names, fmt, watch, w, dcs):
         dcs = get_dcs(config, cluster_name)
 
         for _ in watching(w, watch):
-            output_members(dcs.get_cluster(), cluster_name, fmt)
+            cluster = dcs.get_cluster()
+            output_members(cluster, cluster_name, extended, fmt)
 
 
 def timestamp(precision=6):
@@ -646,3 +730,110 @@ def configure(config_file, dcs, namespace):
     config['dcs_api'] = str(dcs)
     config['namespace'] = str(namespace)
     store_config(config, config_file)
+
+
+def touch_member(config, dcs):
+    ''' Rip-off of the ha.touch_member without inter-class dependencies '''
+    p = Postgresql(config['postgresql'])
+    p.set_state('running')
+    p.set_role('master')
+
+    def restapi_connection_string(config):
+        protocol = 'https' if config.get('certfile') else 'http'
+        connect_address = config.get('connect_address')
+        listen = config['listen']
+        return '{0}://{1}/patroni'.format(protocol, connect_address or listen)
+
+    data = {
+        'conn_url': p.connection_string,
+        'api_url': restapi_connection_string(config['restapi']),
+        'state': p.state,
+        'role': p.role
+    }
+
+    return dcs.touch_member(json.dumps(data, separators=(',', ':')), permanent=True)
+
+
+def set_defaults(config, cluster_name):
+    """fill-in some basic configuration parameters if config file is not set """
+    config['postgresql'].setdefault('name', cluster_name)
+    config['postgresql'].setdefault('scope', cluster_name)
+    config['postgresql'].setdefault('listen', '127.0.0.1')
+    config['postgresql']['authentication'] = {'replication': None}
+    config['restapi']['listen'] = ':' in config['restapi']['listen'] and config['restapi']['listen'] or '127.0.0.1:8008'
+
+
+@ctl.command('scaffold', help='Create a structure for the cluster in DCS')
+@click.argument('cluster_name')
+@click.option('--sysid', '-s', help='System ID of the cluster to put into the initialize key', default="")
+@option_config_file
+@option_dcs
+def scaffold(cluster_name, config_file, dcs, sysid):
+    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    if cluster and cluster.initialize is not None:
+        raise PatroniCtlException("This cluster is already initialized")
+
+    if not dcs.initialize(create_new=True, sysid=sysid):
+        # initialize key already exists, don't touch this cluster
+        raise PatroniCtlException("Initialize key for cluster {0} already exists".format(cluster_name))
+
+    set_defaults(config, cluster_name)
+
+    # make sure the leader keys will never expire
+    if not (touch_member(config, dcs) and dcs.attempt_to_acquire_leader(permanent=True)):
+        # we did initialize this cluster, but failed to write the leader or member keys, wipe it down completely.
+        dcs.delete_cluster()
+        raise PatroniCtlException("Unable to install permanent leader for cluster {0}".format(cluster_name))
+    click.echo("Cluster {0} has been created successfully".format(cluster_name))
+
+
+@ctl.command('flush', help='Flush scheduled events')
+@click.argument('cluster_name')
+@click.argument('member_names', nargs=-1)
+@click.argument('target', type=click.Choice(['restart']))
+@click.option('--role', '-r', help='Flush only members with this role', default='any',
+              type=click.Choice(['master', 'replica', 'any']))
+@option_config_file
+@option_force
+@option_dcs
+def flush(cluster_name, member_names, config_file, dcs, force, role, target):
+    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+
+    members = get_members(cluster, cluster_name, member_names, role, force, 'flush')
+    for member in members:
+        if target == 'restart':
+            if member.data.get('scheduled_restart'):
+                r = request_patroni(member, 'delete', 'restart', None, auth_header(config))
+                check_response(r, member.name, 'flush scheduled restart')
+            else:
+                click.echo('No scheduled restart for member {0}'.format(member.name))
+
+
+def toggle_pause(config_file, cluster_name, dcs, paused):
+    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    if cluster.is_paused() == paused:
+        raise PatroniCtlException('Cluster is {0} paused'.format(paused and 'already' or 'not'))
+
+    r = request_patroni(cluster.leader.member, 'patch', 'config', {'pause': paused or None}, auth_header(config))
+
+    if r.status_code == 200:
+        click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
+    else:
+        click.echo('Failed: {0} cluster management status code={1}, ({2})'.format(
+                   paused and 'pause' or 'resume', r.status_code, r.text))
+
+
+@ctl.command('pause', help='Disable auto failover')
+@click.argument('cluster_name')
+@option_config_file
+@option_dcs
+def pause(config_file, cluster_name, dcs):
+    return toggle_pause(config_file, cluster_name, dcs, True)
+
+
+@ctl.command('resume', help='Resume auto failover')
+@click.argument('cluster_name')
+@option_config_file
+@option_dcs
+def resume(config_file, cluster_name, dcs):
+    return toggle_pause(config_file, cluster_name, dcs, False)

patroni/dcs/__init__.py

@@ -4,7 +4,9 @@ import importlib
 import inspect
 import json
 import os
+import pkgutil
 import six
+import sys
 
 from collections import namedtuple
 from patroni.exceptions import PatroniException
@@ -30,23 +32,37 @@ def parse_connection_string(value):
     return conn_url, api_url
 
 
+def dcs_modules():
+    """Get names of DCS modules, depending on execution environment. If being packaged with PyInstaller,
+    modules aren't discoverable dynamically by scanning source directory because `FrozenImporter` doesn't
+    implement `iter_modules` method. But it is still possible to find all potential DCS modules by
+    iterating through `toc`, which contains list of all "frozen" resources."""
+
+    dcs_dirname = os.path.dirname(__file__)
+    module_prefix = __package__ + '.'
+
+    if getattr(sys, 'frozen', False):
+        importer = pkgutil.get_importer(dcs_dirname)
+        return [module for module in list(importer.toc) if module.startswith(module_prefix) and module.count('.') == 2]
+    else:
+        return [module_prefix + name for _, name, is_pkg in pkgutil.iter_modules([dcs_dirname]) if not is_pkg]
+
+
 def get_dcs(config):
     available_implementations = set()
-    for module in os.listdir(os.path.dirname(__file__)):
+    for module_name in dcs_modules():
-        if module.endswith('.py') and not module.startswith('__'):  # find module
+        module = importlib.import_module(module_name)
-            module_name = module[:-3].lower()
+        for name in filter(lambda name: not name.startswith('__'), dir(module)):  # iterate through module content
-            module = importlib.import_module(__package__ + '.' + module[:-3])
+            value = getattr(module, name)
-            for name in filter(lambda name: not name.startswith('__'), dir(module)):  # iterate through module content
+            name = name.lower()
-                value = getattr(module, name)
+            # try to find implementation of AbstractDCS interface, class name must match with module_name
-                name = name.lower()
+            if inspect.isclass(value) and issubclass(value, AbstractDCS) and __package__ + '.' + name == module_name:
-                # try to find implementation of AbstractDCS interface, class name must match with module_name
+                available_implementations.add(name)
-                if inspect.isclass(value) and issubclass(value, AbstractDCS) and name == module_name:
+                if name in config:  # which has configuration section in the config file
-                    available_implementations.add(name)
+                    # propagate some parameters
-                    if name in config:  # which has configuration section in the config file
+                    config[name].update({p: config[p] for p in ('namespace', 'name', 'scope',
-                        # propagate some parameters
+                                         'loop_wait', 'ttl', 'retry_timeout') if p in config})
-                        config[name].update({p: config[p] for p in ('namespace', 'name',
+                    return value(config[name])
-                                             'scope', 'ttl', 'retry_timeout') if p in config})
-                        return value(config[name])
     raise PatroniException("""Can not find suitable configuration of distributed configuration store
 Available implementations: """ + ', '.join(available_implementations))
 
@@ -86,6 +102,26 @@ class Member(namedtuple('Member', 'index,name,session,data')):
     def conn_url(self):
         return self.data.get('conn_url')
 
+    def conn_kwargs(self, auth=None):
+        ret = self.data.get('conn_kwargs')
+        if ret:
+            ret = ret.copy()
+        else:
+            r = urlparse(self.conn_url)
+            ret = {
+                'host': r.hostname,
+                'port': r.port or 5432,
+                'database': r.path[1:]
+            }
+            self.data['conn_kwargs'] = ret.copy()
+
+        if auth and isinstance(auth, dict):
+            if 'username' in auth:
+                ret['user'] = auth['username']
+            if 'password' in auth:
+                ret['password'] = auth['password']
+        return ret
+
     @property
     def api_url(self):
         return self.data.get('api_url')
@@ -104,7 +140,7 @@ class Member(namedtuple('Member', 'index,name,session,data')):
 
     @property
     def clonefrom(self):
-        return self.tags.get('clonefrom', False)
+        return self.tags.get('clonefrom', False) and bool(self.conn_url)
 
 
 class Leader(namedtuple('Leader', 'index,session,member')):
@@ -119,6 +155,9 @@ class Leader(namedtuple('Leader', 'index,session,member')):
     def name(self):
         return self.member.name
 
+    def conn_kwargs(self, auth=None):
+        return self.member.conn_kwargs(auth)
+
     @property
     def conn_url(self):
         return self.member.conn_url
@@ -163,6 +202,9 @@ class Failover(namedtuple('Failover', 'index,leader,candidate,scheduled_at')):
 
         return Failover(index, data.get('leader'), data.get('member'), data.get('scheduled_at'))
 
+    def __len__(self):
+        return int(bool(self.leader)) + int(bool(self.candidate))
+
 
 class ClusterConfig(namedtuple('ClusterConfig', 'index,data,modify_index')):
 
@@ -205,6 +247,9 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operat
         candidates = [m for m in self.members if m.clonefrom and (not self.leader or m.name != self.leader.name)]
         return candidates[randint(0, len(candidates) - 1)] if candidates else self.leader
 
+    def is_paused(self):
+        return self.config and self.config.data.get('pause', False) or False
+
 
 @six.add_metaclass(abc.ABCMeta)
 class AbstractDCS(object):
@@ -225,6 +270,7 @@ class AbstractDCS(object):
         self._name = config['name']
         self._namespace = '/{0}'.format(config.get('namespace', '/service/').strip('/'))
         self._base_path = '/'.join([self._namespace, config['scope']])
+        self._set_loop_wait(config.get('loop_wait', 10))
 
         self._cluster = None
         self._cluster_thread_lock = Lock()
@@ -269,6 +315,18 @@ class AbstractDCS(object):
     def set_retry_timeout(self, retry_timeout):
         """Set the new value for retry_timeout"""
 
+    def _set_loop_wait(self, loop_wait):
+        self._loop_wait = loop_wait
+
+    def reload_config(self, config):
+        self._set_loop_wait(config['loop_wait'])
+        self.set_ttl(config['ttl'])
+        self.set_retry_timeout(config['retry_timeout'])
+
+    @property
+    def loop_wait(self):
+        return self._loop_wait
+
     @abc.abstractmethod
     def _load_cluster(self):
         """Internally this method should build  `Cluster` object which
@@ -313,9 +371,11 @@ class AbstractDCS(object):
         for example for etcd `prevValue` parameter must be used."""
 
     @abc.abstractmethod
-    def attempt_to_acquire_leader(self):
+    def attempt_to_acquire_leader(self, permanent=False):
         """Attempt to acquire leader lock
         This method should create `/leader` key with value=`~self._name`
+        :param permanent: if set to `!True`, the leader key will never expire.
+         Used in patronictl for the external master
         :returns: `!True` if key has been created successfully.
 
         Key must be created atomically. In case if key already exists it should not be
@@ -343,13 +403,15 @@ class AbstractDCS(object):
         """Create or update `/config` key"""
 
     @abc.abstractmethod
-    def touch_member(self, data, ttl=None):
+    def touch_member(self, data, ttl=None, permanent=False):
         """Update member key in DCS.
         This method should create or update key with the name = '/members/' + `~self._name`
         and value = data in a given DCS.
 
         :param data: json serialized information about instance (including connection strings)
         :param ttl: ttl for member key, optional parameter. If it is None `~self.member_ttl will be used`
+        :param permanent: if set to `!True`, the member key will never expire.
+         Used in patronictl for the external master.
         :returns: `!True` on success otherwise `!False`
         """
 

patroni/dcs/consul.py

@@ -171,7 +171,7 @@ class Consul(AbstractDCS):
 
             self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover)
         except NotFound:
-            self._cluster = Cluster(False, None, None, None, [], None)
+            self._cluster = Cluster(None, None, None, None, [], None)
         except:
             logger.exception('get_cluster')
             raise ConsulError('Consul is not responding properly')
@@ -191,7 +191,8 @@ class Consul(AbstractDCS):
             return True
 
         try:
-            self._client.kv.put(self.member_path, data, acquire=self._session)
+            args = {} if kwargs.get('permanent', False) else {'acquire': self._session}
+            self._client.kv.put(self.member_path, data, **args)
             self._my_member_data = data
             return True
         except Exception:
@@ -199,8 +200,9 @@ class Consul(AbstractDCS):
         return False
 
     @catch_consul_errors
-    def attempt_to_acquire_leader(self):
+    def attempt_to_acquire_leader(self, permanent=False):
-        ret = self._client.kv.put(self.leader_path, self._name, acquire=self._session)
+        args = {} if permanent else {'acquire': self._session}
+        ret = self._client.kv.put(self.leader_path, self._name, **args)
         if not ret:
             logger.info('Could not take out TTL lock')
         return ret

patroni/dcs/etcd.py

@@ -31,24 +31,57 @@ class Client(etcd.Client):
         self._load_machines_cache()
         self._allow_reconnect = True
 
+    def _build_request_parameters(self):
+        kwargs = {'headers': self._get_headers(), 'redirect': self.allow_redirect}
+
+        # calculate the number of retries and timeout *per node*
+        # actual number of retries depends on the number of nodes
+        etcd_nodes = len(self._machines_cache) + 1
+        kwargs['retries'] = 0 if etcd_nodes > 3 else (1 if etcd_nodes > 1 else 2)
+
+        # if etcd_nodes > 3:
+        #     kwargs.update({'retries': 0, 'timeout': float(self.read_timeout)/etcd_nodes})
+        # elif etcd_nodes > 1:
+        #     kwargs.update({'retries': 1, 'timeout': self.read_timeout/2.0/etcd_nodes})
+        # else:
+        #     kwargs.update({'retries': 2, 'timeout': self.read_timeout/3.0})
+        kwargs['timeout'] = self.read_timeout/float(kwargs['retries'] + 1)/etcd_nodes
+        return kwargs
+
     @property
     def machines(self):
         """Original `machines` method(property) of `etcd.Client` class raise exception
         when it failed to get list of etcd cluster members. This method is being called
         only when request failed on one of the etcd members during `api_execute` call.
-        For us it's more important to execute original request rather then get new
+        For us it's more important to execute original request rather then get new topology
-        topology of etcd cluster. So we will catch this exception and return valid list
+        of etcd cluster. So we will catch this exception and return empty list of machines.
-        of machines with setting flag `self._update_machines_cache` to `!True`.
+        Later, during next `api_execute` call we will forcefully update machines_cache.
-        Later, during next `api_execute` call we will forcefully update machines_cache"""
+
-        try:
+        Also this method implements the same timeout-retry logic as `api_execute`, because
-            ret = super(Client, self).machines
+        the original method was retrying 2 times with the `read_timeout` on each node."""
-            random.shuffle(ret)
+
-            return ret
+        kwargs = self._build_request_parameters()
-        except etcd.EtcdException:
+
-            if self._update_machines_cache:  # We are updating machines_cache
+        while True:
-                raise  # This exception is fatal, we should re-raise it.
+            try:
-            self._update_machines_cache = True
+                response = self.http.request(self._MGET, self._base_uri + self.version_prefix + '/machines', **kwargs)
-            return [self._base_uri]
+                machines = [n.strip() for n in self._handle_server_response(response).data.decode('utf-8').split(',')]
+                logger.debug("Retrieved list of machines: %s", machines)
+                random.shuffle(machines)
+                return machines
+            except Exception as e:
+                # We can't get the list of machines, if one server is in the
+                # machines cache, try on it
+                logger.error("Failed to get list of machines from %s%s: %r", self._base_uri, self.version_prefix, e)
+                if self._machines_cache:
+                    self._base_uri = self._machines_cache.pop(0)
+                    logger.info("Retrying on %s", self._base_uri)
+                elif self._update_machines_cache:
+                    raise etcd.EtcdException("Could not get the list of servers, "
+                                             "maybe you provided the wrong "
+                                             "host(s) to connect to?")
+                else:
+                    return []
 
     def set_read_timeout(self, timeout):
         self._read_timeout = timeout
@@ -73,8 +106,7 @@ class Client(etcd.Client):
         if not path.startswith('/'):
             raise ValueError('Path does not start with /')
 
-        kwargs = {'fields': params, 'redirect': self.allow_redirect,
+        kwargs = {'fields': params, 'preload_content': False}
-                  'headers': self._get_headers(), 'preload_content': False}
 
         if method in [self._MGET, self._MDELETE]:
             request_executor = self.http.request
@@ -88,35 +120,29 @@ class Client(etcd.Client):
         if self._update_machines_cache:
             self._load_machines_cache()
 
-        if timeout is None:
+        kwargs.update(self._build_request_parameters())
-            # calculate the number of retries and timeout *per node*
-            # actual number of retries depends on the number of nodes
-            etcd_nodes = len(self._machines_cache) + 1
-            kwargs['retries'] = 0 if etcd_nodes > 3 else (1 if etcd_nodes > 1 else 2)
 
-            # if etcd_nodes > 3:
+        if timeout is not None:
-            #     kwargs.update({'retries': 0, 'timeout': float(self.read_timeout)/etcd_nodes})
-            # elif etcd_nodes > 1:
-            #     kwargs.update({'retries': 1, 'timeout': self.read_timeout/2.0/etcd_nodes})
-            # else:
-            #     kwargs.update({'retries': 2, 'timeout': self.read_timeout/3.0})
-            kwargs['timeout'] = self.read_timeout/float(kwargs['retries'] + 1)/etcd_nodes
-        else:
             kwargs.update({'retries': 0, 'timeout': timeout})
 
         response = False
 
         try:
+            some_request_failed = False
             while not response:
                 response = self._do_http_request(request_executor, method, self._base_uri + path, **kwargs)
 
-                if response is False and not self._use_proxies:
+                if response is False:
-                    self._machines_cache = self.machines
+                    some_request_failed = True
+            if some_request_failed and not self._use_proxies:
+                self._machines_cache = self.machines
+                if self._base_uri in self._machines_cache:
                     self._machines_cache.remove(self._base_uri)
-            return self._handle_server_response(response)
         except etcd.EtcdConnectionFailed:
             self._update_machines_cache = True
-            raise
+            if not response:
+                raise
+        return self._handle_server_response(response)
 
     @staticmethod
     def get_srv_record(host):
@@ -266,7 +292,8 @@ class Etcd(AbstractDCS):
             if leader:
                 member = Member(-1, leader.value, None, {})
                 member = ([m for m in members if m.name == leader.value] or [member])[0]
-                leader = Leader(leader.modifiedIndex, leader.ttl, member)
+                index = result.etcd_index if result.etcd_index > leader.modifiedIndex else leader.modifiedIndex + 1
+                leader = Leader(index, leader.ttl, member)
 
             # failover key
             failover = nodes.get(self._FAILOVER)
@@ -275,22 +302,26 @@ class Etcd(AbstractDCS):
 
             self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover)
         except etcd.EtcdKeyNotFound:
-            self._cluster = Cluster(False, None, None, None, [], None)
+            self._cluster = Cluster(None, None, None, None, [], None)
         except:
             logger.exception('get_cluster')
             raise EtcdError('Etcd is not responding properly')
 
     @catch_etcd_errors
-    def touch_member(self, data, ttl=None):
+    def touch_member(self, data, ttl=None, permanent=False):
-        return self.retry(self._client.set, self.member_path, data, ttl or self._ttl)
+        return self.retry(self._client.set, self.member_path, data, None if permanent else ttl or self._ttl)
 
     @catch_etcd_errors
     def take_leader(self):
         return self.retry(self._client.set, self.leader_path, self._name, self._ttl)
 
-    def attempt_to_acquire_leader(self):
+    def attempt_to_acquire_leader(self, permanent=False):
         try:
-            return bool(self.retry(self._client.write, self.leader_path, self._name, ttl=self._ttl, prevExist=False))
+            return bool(self.retry(self._client.write,
+                                   self.leader_path,
+                                   self._name,
+                                   ttl=None if permanent else self._ttl,
+                                   prevExist=False))
         except etcd.EtcdAlreadyExist:
             logger.info('Could not take out TTL lock')
         except (RetryFailedError, etcd.EtcdException):
@@ -341,7 +372,7 @@ class Etcd(AbstractDCS):
 
             while timeout >= 1:  # when timeout is too small urllib3 doesn't have enough time to connect
                 try:
-                    self._client.watch(self.leader_path, index=cluster.leader.index + 1, timeout=timeout + 0.5)
+                    self._client.watch(self.leader_path, index=cluster.leader.index, timeout=timeout + 0.5)
                     # Synchronous work of all cluster members with etcd is less expensive
                     # than reestablishing http connection every time from every replica.
                     return True

patroni/dcs/zookeeper.py

@@ -20,7 +20,7 @@ class PatroniSequentialThreadingHandler(SequentialThreadingHandler):
         self.set_connect_timeout(connect_timeout)
 
     def set_connect_timeout(self, connect_timeout):
-        self._connect_timeout = max(1.0, connect_timeout/4.0)
+        self._connect_timeout = max(1.0, connect_timeout/2.0)  # try to connect to zookeeper node during loop_wait/2
 
     def create_connection(self, *args, **kwargs):
         """This method is trying to establish connection with one of the zookeeper nodes.
@@ -59,8 +59,27 @@ class ZooKeeper(AbstractDCS):
         self._fetch_cluster = True
         self._last_leader_operation = 0
 
+        self._orig_kazoo_connect = self._client._connection._connect
+        self._client._connection._connect = self._kazoo_connect
+
         self._client.start()
 
+    def _kazoo_connect(self, host, port):
+
+        """Kazoo is using Ping's to determine health of connection to zookeeper. If there is no
+        response on Ping after Ping interval (1/2 from read_timeout) it will consider current
+        connection dead and try to connect to another node. Without this "magic" it was taking
+        up to 2/3 from session timeout (ttl) to figure out that connection was dead and we had
+        only small time for reconnect and retry.
+
+        This method is needed to return different value of read_timeout, which is not calculated
+        from negotiated session timeout but from value of `loop_wait`. And it is 2 sec smaller
+        than loop_wait, because we can spend up to 2 seconds when calling `touch_member()` and
+        `write_leader_optime()` methods, which also may hang..."""
+
+        ret = self._orig_kazoo_connect(host, port)
+        return max(self.loop_wait - 2, 2)*1000, ret[1]
+
     def session_listener(self, state):
         if state in [KazooState.SUSPENDED, KazooState.LOST]:
             self.cluster_watcher(None)
@@ -69,15 +88,34 @@ class ZooKeeper(AbstractDCS):
         self._fetch_cluster = True
         self.event.set()
 
+    def reload_config(self, config):
+        self.set_retry_timeout(config['retry_timeout'])
+
+        loop_wait = config['loop_wait']
+
+        loop_wait_changed = self._loop_wait != loop_wait
+        self._loop_wait = loop_wait
+        self._client.handler.set_connect_timeout(loop_wait)
+
+        # We need to reestablish connection to zookeeper if we want to change
+        # read_timeout (and Ping interval respectively), because read_timeout
+        # is calculated in `_kazoo_connect` method. If we are changing ttl at
+        # the same time, set_ttl method will reestablish connection and return
+        # `!True`, otherwise we will close existing connection and let kazoo
+        # open the new one.
+        if not self.set_ttl(int(config['ttl'] * 1000)) and loop_wait_changed:
+            self._client._connection._socket.close()
+
     def set_ttl(self, ttl):
-        ttl = int(ttl * 1000)
+        """It is not possible to change ttl (session_timeout) in zookeeper without
-        # I know, it's weird to access private attributes
+        destroying old session and creating the new one. This method returns `!True`
+        if session_timeout has been changed (`restart()` has been called)."""
         if self._client._session_timeout != ttl:
             self._client._session_timeout = ttl
             self._client.restart()
+            return True
 
     def set_retry_timeout(self, retry_timeout):
-        self._client.handler.set_connect_timeout(retry_timeout)
         self._client._retry.deadline = retry_timeout
 
     def get_node(self, key, watch=None):
@@ -150,7 +188,7 @@ class ZooKeeper(AbstractDCS):
         if self._fetch_cluster or self._cluster is None:
             try:
                 self._client.retry(self._inner_load_cluster)
-            except:
+            except Exception:
                 logger.exception('get_cluster')
                 self.cluster_watcher(None)
                 raise ZooKeeperError('ZooKeeper in not responding properly')
@@ -162,8 +200,8 @@ class ZooKeeper(AbstractDCS):
         except:
             return False
 
-    def attempt_to_acquire_leader(self):
+    def attempt_to_acquire_leader(self, permanent=False):
-        ret = self._create(self.leader_path, self._name, makepath=True, ephemeral=True)
+        ret = self._create(self.leader_path, self._name, makepath=True, ephemeral=not permanent)
         if not ret:
             logger.info('Could not take out TTL lock')
         return ret
@@ -192,39 +230,38 @@ class ZooKeeper(AbstractDCS):
         return self._create(self.initialize_path, sysid, makepath=True) if create_new \
             else self._client.retry(self._client.set, self.initialize_path,  sysid.encode("utf-8"))
 
-    def touch_member(self, data, ttl=None):
+    def touch_member(self, data, ttl=None, permanent=False):
         cluster = self.cluster
         member = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
-        path = self.member_path
         data = data.encode('utf-8')
         if member and self._client.client_id is not None and member.session != self._client.client_id[0]:
             try:
-                self._client.retry(self._client.delete, path)
+                self._client.delete_async(self.member_path).get(timeout=1)
             except NoNodeError:
                 pass
             except:
                 return False
             member = None
 
-        if member and data == self._my_member_data:
+        if member:
-            return True
+            if data == self._my_member_data:
-
+                return True
-        try:
+        else:
-            if member:
-                self._client.retry(self._client.set, path, data)
-            else:
-                self._client.retry(self._client.create, path, data, makepath=True, ephemeral=True)
-            self._my_member_data = data
-            return True
-        except NodeExistsError:
             try:
-                self._client.retry(self._client.set, path, data)
+                self._client.create_async(self.member_path, data, makepath=True, ephemeral=not permanent).get(timeout=1)
                 self._my_member_data = data
                 return True
-            except:
+            except Exception as e:
-                logger.exception('touch_member')
+                if not isinstance(e, NodeExistsError):
+                    logger.exception('touch_member')
+                    return False
+        try:
+            self._client.set_async(self.member_path, data).get(timeout=1)
+            self._my_member_data = data
+            return True
         except:
             logger.exception('touch_member')
+
         return False
 
     def take_leader(self):
@@ -233,17 +270,17 @@ class ZooKeeper(AbstractDCS):
     def write_leader_optime(self, last_operation):
         last_operation = last_operation.encode('utf-8')
         if last_operation != self._last_leader_operation:
-            self._last_leader_operation = last_operation
-            path = self.leader_optime_path
             try:
-                self._client.retry(self._client.set, path, last_operation)
+                self._client.set_async(self.leader_optime_path, last_operation).get(timeout=1)
+                self._last_leader_operation = last_operation
             except NoNodeError:
                 try:
-                    self._client.retry(self._client.create, path, last_operation, makepath=True)
+                    self._client.create_async(self.leader_optime_path, last_operation, makepath=True).get(timeout=1)
+                    self._last_leader_operation = last_operation
                 except:
-                    logger.exception('Failed to create %s', path)
+                    logger.exception('Failed to create %s', self.leader_optime_path)
             except:
-                logger.exception('Failed to update %s', path)
+                logger.exception('Failed to update %s', self.leader_optime_path)
 
     def update_leader(self):
         return True

patroni/ha.py

@@ -9,6 +9,7 @@ import pytz
 from multiprocessing.pool import ThreadPool
 from patroni.async_executor import AsyncExecutor
 from patroni.exceptions import DCSError, PostgresConnectionException
+from patroni.postgresql import ACTION_ON_START
 from patroni.utils import sleep
 
 logger = logging.getLogger(__name__)
@@ -25,6 +26,9 @@ class Ha(object):
         self.recovering = False
         self._async_executor = AsyncExecutor()
 
+    def is_paused(self):
+        return self.cluster and self.cluster.is_paused()
+
     def load_cluster_from_dcs(self):
         cluster = self.dcs.get_cluster()
 
@@ -66,6 +70,11 @@ class Ha(object):
                 data['xlog_location'] = self.state_handler.xlog_position()
             except:
                 pass
+        if self.patroni.scheduled_restart:
+            scheduled_restart_data = self.patroni.scheduled_restart.copy()
+            scheduled_restart_data['schedule'] = scheduled_restart_data['schedule'].isoformat()
+            data['scheduled_restart'] = scheduled_restart_data
+
         self.dcs.touch_member(json.dumps(data, separators=(',', ':')))
 
     def clone(self, clone_member=None, msg='(without leader)'):
@@ -125,24 +134,34 @@ class Ha(object):
 
         return node_to_follow if node_to_follow and node_to_follow.name != self.state_handler.name else None
 
-    def follow(self, demote_reason, follow_reason, refresh=True, recovery=False):
+    def follow(self, demote_reason, follow_reason, refresh=True, recovery=False, need_rewind=None):
         if refresh:
             self.load_cluster_from_dcs()
 
         if recovery:
             ret = demote_reason if self.has_lock() else follow_reason
         else:
-            ret = demote_reason if self.state_handler.is_leader() else follow_reason
+            is_leader = self.state_handler.is_leader()
+            ret = demote_reason if is_leader else follow_reason
 
         node_to_follow = self._get_node_to_follow(self.cluster)
 
-        if not self.state_handler.check_recovery_conf(node_to_follow) or recovery:
+        if self.is_paused() and not self.state_handler.need_rewind:
-            self._async_executor.schedule('changing primary_conninfo and restarting')
+            self.state_handler.set_role('master' if is_leader else 'replica')
-            self._async_executor.run_async(self.state_handler.follow, (node_to_follow, self.cluster.leader, recovery))
+            if is_leader:
+                return 'continue to run as master without lock'
+            elif not node_to_follow:
+                return 'no action'
+
+        self.state_handler.follow(node_to_follow, self.cluster.leader, recovery, self._async_executor, need_rewind)
+
         return ret
 
     def enforce_master_role(self, message, promote_message):
         if self.state_handler.is_leader() or self.state_handler.role == 'master':
+            # Inform the state handler about its master role.
+            # It may be unaware of it if postgres is promoted manually.
+            self.state_handler.set_role('master')
             return message
         else:
             self.state_handler.promote()
@@ -167,8 +186,8 @@ class Ha(object):
             is_master = json['role'] == 'master'
             xlog_location = None if is_master else json['xlog']['replayed_location']
             return (member, True, not is_master, xlog_location, json.get('tags', {}))
-        except:
+        except Exception as e:
-            logging.exception('request failed: GET %s', member.api_url)
+            logger.warning("request failed: GET %s (%s)", member.api_url, e)
         return (member, False, None, 0, {})
 
     def fetch_nodes_statuses(self, members):
@@ -218,6 +237,15 @@ class Ha(object):
         if failover.candidate:  # manual failover to specific member
             if failover.candidate == self.state_handler.name:  # manual failover to me
                 return True
+            elif self.is_paused():
+                # Remove failover key if the node to failover has terminated to avoid waiting for it indefinitely
+                # In order to avoid attempts to delete this key from all nodes only the master is allowed to do it.
+                if (not self.cluster.get_member(failover.candidate, fallback_to_leader=False) and
+                   self.state_handler.is_leader()):
+                    logger.warning("manual failover: removing failover key because failover candidate is not running")
+                    self.dcs.manual_failover('', '', index=self.cluster.failover.index)
+                    return None
+                return False
 
             # find specific node and check that it is healthy
             member = self.cluster.get_member(failover.candidate, fallback_to_leader=False)
@@ -234,6 +262,8 @@ class Ha(object):
 
             # at this point we should consider all members as a candidates for failover
             # i.e. we assume that failover.candidate is None
+        elif self.is_paused():
+            return False
 
         # try to pick some other members to failover and check that they are healthy
         if failover.leader:
@@ -252,9 +282,18 @@ class Ha(object):
         return self._is_healthiest_node(members, check_replication_lag=False)
 
     def is_healthiest_node(self):
+        if self.is_paused() and not self.patroni.nofailover and \
+                self.cluster.failover and not self.cluster.failover.scheduled_at:
+            ret = self.manual_failover_process_no_leader()
+            if ret is not None:  # continue if we just deleted the stale failover key as a master
+                return ret
+
         if self.state_handler.is_leader():  # leader is always the healthiest
             return True
 
+        if self.is_paused():
+            return False
+
         if self.patroni.nofailover:  # nofailover tag makes node always unhealthy
             return False
 
@@ -268,67 +307,91 @@ class Ha(object):
     def demote(self, delete_leader=True):
         if delete_leader:
             self.state_handler.stop()
-            self.state_handler.set_role('unknown')
+            self.state_handler.set_role('demoted')
             self.dcs.delete_leader()
             self.touch_member()
             self.dcs.reset_cluster()
-            sleep(2)  # Give a time to somebody to promote
+            sleep(2)  # Give a time to somebody to take the leader lock
             cluster = self.dcs.get_cluster()
             node_to_follow = self._get_node_to_follow(cluster)
-            self.state_handler.follow(node_to_follow, cluster.leader, True)
+            self.state_handler.follow(node_to_follow, cluster.leader, recovery=True, need_rewind=True)
         else:
             self.state_handler.follow(None, None)
 
+    def should_run_scheduled_action(self, action_name, scheduled_at, cleanup_fn):
+        if scheduled_at and not self.is_paused():
+            # If the scheduled action is in the far future, we shouldn't do anything and just return.
+            # If the scheduled action is in the past, we consider the value to be stale and we remove
+            # the value.
+            # If the value is close to now, we initiate the scheduled action
+            # Additionally, if the scheduled action cannot be executed altogether, i.e. there is an error
+            # or the action is in the past - we take care of cleaning it up.
+            now = datetime.datetime.now(pytz.utc)
+            try:
+                delta = (scheduled_at - now).total_seconds()
+
+                if delta > self.dcs.loop_wait:
+                    logger.info('Awaiting %s at %s (in %.0f seconds)',
+                                action_name, scheduled_at.isoformat(), delta)
+                    return False
+                elif delta < - int(self.dcs.loop_wait * 1.5):
+                    logger.warning('Found a stale %s value, cleaning up: %s',
+                                   action_name, scheduled_at.isoformat())
+                    cleanup_fn()
+                    return False
+
+                # The value is very close to now
+                sleep(max(delta, 0))
+                logger.info('Manual scheduled {0} at %s'.format(action_name), scheduled_at.isoformat())
+                return True
+            except TypeError:
+                logger.warning('Incorrect value of scheduled_at: %s', scheduled_at)
+                cleanup_fn()
+        return False
+
     def process_manual_failover_from_leader(self):
         failover = self.cluster.failover
 
-        if failover.scheduled_at:
+        if (failover.scheduled_at and not
-            # If the failover is in the far future, we shouldn't do anything and just return.
+            self.should_run_scheduled_action("failover", failover.scheduled_at, lambda:
-            # If the failover is in the past, we consider the value to be stale and we remove
+                                             self.dcs.manual_failover('', '', index=failover.index))):
-            # the value.
+            return
-            # If the value is close to now, we initiate the failover
-            now = datetime.datetime.now(pytz.utc)
-            try:
-                delta = (failover.scheduled_at - now).total_seconds()
-
-                if delta > self.patroni.nap_time:
-                    logging.info('Awaiting failover at %s (in %.0f seconds)', failover.scheduled_at.isoformat(), delta)
-                    return
-                elif delta < - int(self.patroni.nap_time * 1.5):
-                    logger.warning('Found a stale failover value, cleaning up: %s', failover.scheduled_at)
-                    self.dcs.manual_failover('', '', index=self.cluster.failover.index)
-                    return
-
-                # The value is very close to now
-                sleep(max(delta, 0))
-                logger.info('Manual scheduled failover at {}'.format(failover.scheduled_at.isoformat()))
-            except TypeError:
-                logger.warning('Incorrect value in of scheduled_at: %s', failover.scheduled_at)
 
         if not failover.leader or failover.leader == self.state_handler.name:
             if not failover.candidate or failover.candidate != self.state_handler.name:
-                members = [m for m in self.cluster.members if not failover.candidate or m.name == failover.candidate]
+                if not failover.candidate and self.is_paused():
-                if self.is_failover_possible(members):  # check that there are healthy members
+                    logger.warning('Failover is possible only to a specific candidate in a paused state')
-                    self._async_executor.schedule('manual failover: demote')
-                    self._async_executor.run_async(self.demote)
-                    return 'manual failover: demoting myself'
                 else:
-                    logger.warning('manual failover: no healthy members found, failover is not possible')
+                    members = [m for m in self.cluster.members
+                               if not failover.candidate or m.name == failover.candidate]
+                    if self.is_failover_possible(members):  # check that there are healthy members
+                        self._async_executor.schedule('manual failover: demote')
+                        self._async_executor.run_async(self.demote)
+                        return 'manual failover: demoting myself'
+                    else:
+                        logger.warning('manual failover: no healthy members found, failover is not possible')
             else:
                 logger.warning('manual failover: I am already the leader, no need to failover')
         else:
             logger.warning('manual failover: leader name does not match: %s != %s',
-                           self.cluster.failover.leader, self.state_handler.name)
+                           failover.leader, self.state_handler.name)
 
-        logger.info('Trying to clean up failover key')
+        logger.info('Cleaning up failover key')
-        self.dcs.manual_failover('', '', index=self.cluster.failover.index)
+        self.dcs.manual_failover('', '', index=failover.index)
 
     def process_unhealthy_cluster(self):
+        """Cluster has no leader key"""
+
         if self.is_healthiest_node():
             if self.acquire_lock():
-                if self.cluster.failover:
+                failover = self.cluster.failover
-                    logger.info('Cleaning up failover key after acquiring leader lock...')
+                if failover:
-                    self.dcs.manual_failover('', '')
+                    if self.is_paused() and failover.leader and failover.candidate:
+                        logger.info('Updating failover key after acquiring leader lock...')
+                        self.dcs.manual_failover('', failover.candidate, failover.scheduled_at, failover.index)
+                    else:
+                        logger.info('Cleaning up failover key after acquiring leader lock...')
+                        self.dcs.manual_failover('', '')
                 self.load_cluster_from_dcs()
                 return self.enforce_master_role('acquired session lock as a leader',
                                                 'promoted self to leader by acquiring session lock')
@@ -336,19 +399,35 @@ class Ha(object):
                 return self.follow('demoted self after trying and failing to obtain lock',
                                    'following new leader after trying and failing to obtain lock')
         else:
+            # when we are doing manual failover there is no guaranty that new leader is ahead of any other node
+            # node tagged as nofailover can be ahead of the new leader either, but it is always excluded from elections
+            need_rewind = bool(self.cluster.failover) or self.patroni.nofailover
+            if need_rewind:
+                sleep(2)  # Give a time to somebody to take the leader lock
+
             if self.patroni.nofailover:
                 return self.follow('demoting self because I am not allowed to become master',
-                                   'following a different leader because I am not allowed to promote')
+                                   'following a different leader because I am not allowed to promote',
+                                   need_rewind=need_rewind)
             return self.follow('demoting self because i am not the healthiest node',
-                               'following a different leader because i am not the healthiest node')
+                               'following a different leader because i am not the healthiest node',
+                               need_rewind=need_rewind)
 
     def process_healthy_cluster(self):
         if self.has_lock():
-            if self.cluster.failover:
+            if self.cluster.failover and (not self.is_paused() or self.state_handler.is_leader()):
                 msg = self.process_manual_failover_from_leader()
                 if msg is not None:
                     return msg
 
+            if self.is_paused() and not self.state_handler.is_leader():
+                if self.cluster.failover and self.cluster.failover.candidate == self.state_handler.name:
+                    return 'waiting to become master after promote...'
+
+                self.dcs.delete_leader()
+                self.dcs.reset_cluster()
+                return 'removed leader lock because postgres is not running as master'
+
             if self.update_lock():
                 return self.enforce_master_role('no action.  i am the leader with the lock',
                                                 'promoted self to leader because i had the session lock')
@@ -361,48 +440,117 @@ class Ha(object):
         return self.follow('demoting self because i do not have the lock and i was a leader',
                            'no action.  i am a secondary and i am following a leader', False)
 
-    def schedule(self, action):
+    def evaluate_scheduled_restart(self):
+        # restart if we need to
+        restart_data = self.future_restart_scheduled()
+        if restart_data:
+            recent_time = self.state_handler.postmaster_start_time()
+            request_time = restart_data['postmaster_start_time']
+            # check if postmaster start time has changed since the last restart
+            if recent_time and request_time and recent_time != request_time:
+                logger.info("Cancelling scheduled restart: postgres restart has already happened at %s", recent_time)
+                self.delete_future_restart()
+                return None
+
+        if (restart_data and
+           self.should_run_scheduled_action('restart', restart_data['schedule'], self.delete_future_restart)):
+            try:
+                ret, message = self.restart(restart_data, run_async=True)
+                if not ret:
+                    logger.warning("Scheduled restart: %s", message)
+                    return None
+                return message
+            finally:
+                self.delete_future_restart()
+
+    def restart_matches(self, role, postgres_version, pending_restart):
+        reason_to_cancel = ""
+        # checking the restart filters here seem to be less ugly than moving them into the
+        # run_scheduled_action.
+        if role and role != self.state_handler.role:
+            reason_to_cancel = "host role mismatch"
+
+        if (postgres_version and
+           self.state_handler.postgres_version_to_int(postgres_version) <= int(self.state_handler.server_version)):
+            reason_to_cancel = "postgres version mismatch"
+
+        if pending_restart and not self.state_handler.pending_restart:
+            reason_to_cancel = "pending restart flag is not set"
+
+        if not reason_to_cancel:
+            return True
+        else:
+            logger.info("not proceeding with the restart: %s", reason_to_cancel)
+        return False
+
+    def schedule_future_restart(self, restart_data):
         with self._async_executor:
-            return self._async_executor.schedule(action)
+            if not self.patroni.scheduled_restart:
+                self.patroni.scheduled_restart = restart_data
+                self.touch_member()
+                return True
+        return False
+
+    def delete_future_restart(self):
+        ret = False
+        with self._async_executor:
+            if self.patroni.scheduled_restart:
+                self.patroni.scheduled_restart = {}
+                self.touch_member()
+                ret = True
+        return ret
+
+    def future_restart_scheduled(self):
+        return self.patroni.scheduled_restart.copy() if (self.patroni.scheduled_restart and
+                                                         isinstance(self.patroni.scheduled_restart, dict)) else None
 
     def restart_scheduled(self):
         return self._async_executor.scheduled_action == 'restart'
 
-    def schedule_reinitialize(self):
+    def restart(self, restart_data=None, run_async=False):
-        return self.schedule('reinitialize')
+        """ conditional and unconditional restart """
+        if (restart_data and isinstance(restart_data, dict) and
+            not self.restart_matches(restart_data.get('role'),
+                                     restart_data.get('postgres_version'),
+                                     ('restart_pending' in restart_data))):
+            return (False, "restart conditions are not satisfied")
 
-    def reinitialize_scheduled(self):
-        return self._async_executor.scheduled_action == 'reinitialize'
-
-    def restart(self):
         with self._async_executor:
-            prev = self._async_executor.schedule('restart', True)
+            prev = self._async_executor.schedule('restart')
             if prev is not None:
                 return (False, prev + ' already in progress')
-        if self._async_executor.run(self.state_handler.restart):
+
+        if run_async:
+            self._async_executor.run_async(self.state_handler.restart)
+            return (True, 'restart initiated')
+        elif self._async_executor.run(self.state_handler.restart):
             return (True, 'restarted successfully')
         else:
             return (False, 'restart failed')
 
-    def reinitialize(self, cluster):
+    def _do_reinitialize(self, cluster):
         self.state_handler.stop('immediate')
         self.state_handler.remove_data_directory()
 
-        clone_member = cluster.get_clone_member()
+        clone_member = self.cluster.get_clone_member()
-        member_role = 'leader' if clone_member == cluster.leader else 'replica'
+        member_role = 'leader' if clone_member == self.cluster.leader else 'replica'
         self.clone(clone_member, "from {0} '{1}'".format(member_role, clone_member.name))
 
-    def process_scheduled_action(self):
+    def reinitialize(self):
-        if self.reinitialize_scheduled():
+        with self._async_executor:
+            self.load_cluster_from_dcs()
+
             if self.cluster.is_unlocked():
-                logger.error('Cluster has no leader, can not reinitialize')
+                return 'Cluster has no leader, can not reinitialize'
-                self._async_executor.reset_scheduled_action()
+
-            elif self.has_lock():
+            if self.cluster.leader.name == self.state_handler.name:
-                logger.error('I am the leader, can not reinitialize')
+                return 'I am the leader, can not reinitialize'
-                self._async_executor.reset_scheduled_action()
+
-            else:
+            action = self._async_executor.schedule('reinitialize', immediately=True)
-                self._async_executor.run_async(self.reinitialize, args=(self.cluster, ))
+            if action is not None:
-                return 'reinitialize started'
+                return '{0} already in progress'.format(action)
+
+        self._async_executor.run_async(self._do_reinitialize, args=(self.cluster, ))
 
     def handle_long_action_in_progress(self):
         if self.has_lock():
@@ -448,22 +596,17 @@ class Ha(object):
                 return self.handle_long_action_in_progress()
 
             # we've got here, so any async action has finished. Check if we tried to recover and failed
-            if self.recovering:
+            if self.recovering and not self.state_handler.need_rewind:
                 self.recovering = False
                 msg = self.post_recover()
                 if msg is not None:
                     return msg
 
-            # currently it can trigger only reinitialize
-            msg = self.process_scheduled_action()
-            if msg is not None:
-                return msg
-
             # is data directory empty?
             if self.state_handler.data_directory_empty():
                 return self.bootstrap()  # new node
             # "bootstrap", but data directory is not empty
-            elif not self.sysid_valid(self.cluster.initialize) and self.cluster.is_unlocked():
+            elif not self.sysid_valid(self.cluster.initialize) and self.cluster.is_unlocked() and not self.is_paused():
                 self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=self.state_handler.sysid)
             else:
                 # check if we are allowed to join
@@ -472,31 +615,44 @@ class Ha(object):
                                  self.state_handler.name, self.cluster.initialize, self.state_handler.sysid)
                     sys.exit(1)
 
-            # try to start dead postgres
             if not self.state_handler.is_healthy():
-                msg = self.recover()
+                if self.is_paused():
-                if msg is not None:
+                    if self.has_lock():
-                    return msg
+                        self.dcs.delete_leader()
+                        self.dcs.reset_cluster()
+                        return 'removed leader lock because postgres is not running'
+                    elif not self.state_handler.need_rewind:
+                        return 'postgres is not running'
+
+                # try to start dead postgres
+                return self.recover()
 
             try:
                 if self.cluster.is_unlocked():
                     return self.process_unhealthy_cluster()
                 else:
+                    msg = self.evaluate_scheduled_restart()
+                    if msg is not None:
+                        return msg
                     return self.process_healthy_cluster()
             finally:
                 # we might not have a valid PostgreSQL connection here if another thread
                 # stops PostgreSQL, therefore, we only reload replication slots if no
                 # asynchronous processes are running (should be always the case for the master)
                 if not self._async_executor.busy:
+                    if not self.state_handler.cb_called:
+                        self.state_handler.call_nowait(ACTION_ON_START)
                     self.state_handler.sync_replication_slots(self.cluster)
         except DCSError:
             logger.error('Error communicating with DCS')
-            if self.state_handler.is_running() and self.state_handler.is_leader():
+            if not self.is_paused() and self.state_handler.is_running() and self.state_handler.is_leader():
                 self.demote(delete_leader=False)
                 return 'demoted self because DCS is not accessible and i was a leader'
+            return 'DCS is not accessible'
         except (psycopg2.Error, PostgresConnectionException):
-            logger.exception('Error communicating with PostgreSQL. Will try again later')
+            return 'Error communicating with PostgreSQL. Will try again later'
 
     def run_cycle(self):
         with self._async_executor:
-            return self._run_cycle()
+            info = self._run_cycle()
+            return (self.is_paused() and 'PAUSE: ' or '') + info

patroni/postgresql.py

@@ -1,6 +1,8 @@
+from collections import defaultdict
 import logging
 import os
 import psycopg2
+import re
 import shlex
 import shutil
 import subprocess
@@ -10,7 +12,6 @@ import time
 from patroni.exceptions import PostgresConnectionException, PostgresException
 from patroni.utils import compare_values, parse_bool, parse_int, Retry, RetryFailedError
 from six import string_types
-from six.moves.urllib_parse import urlparse
 from threading import Lock
 
 logger = logging.getLogger(__name__)
@@ -22,22 +23,20 @@ ACTION_ON_RELOAD = "on_reload"
 ACTION_ON_ROLE_CHANGE = "on_role_change"
 
 
-def get_conn_kwargs(url, auth=None):
+def slot_name_from_member_name(member_name):
-    r = urlparse(url)
+    """Translate member name to valid PostgreSQL slot name.
-    ret = {
+
-        'host': r.hostname,
+    PostgreSQL replication slot names must be valid PostgreSQL names. This function maps the wider space of
-        'port': r.port or 5432,
+    member names to valid PostgreSQL names. Names are lowercased, dashes and periods common in hostnames
-        'database': r.path[1:],
+    are replaced with underscores, other characters are encoded as their unicode codepoint. Name is truncated
-        'fallback_application_name': 'Patroni',
+    to 64 characters. Multiple different member names may map to a single slot name."""
-        'connect_timeout': 3,
+
-        'options': '-c statement_timeout=2000',
+    def replace_char(match):
-    }
+        c = match.group(0)
-    if auth and isinstance(auth, dict):
+        return '_' if c in '-.' else "u{:04d}".format(ord(c))
-        if 'username' in auth:
+
-            ret['user'] = auth['username']
+    slot_name = re.sub('[^a-z0-9_]', replace_char, member_name.lower())
-        if 'password' in auth:
+    return slot_name[0:64]
-            ret['password'] = auth['password']
-    return ret
 
 
 class Postgresql(object):
@@ -77,6 +76,7 @@ class Postgresql(object):
         self.config = config
         self.name = config['name']
         self.scope = config['scope']
+        self._bin_dir = config.get('bin_dir') or ''
         self._database = config.get('database', 'postgres')
         self._data_dir = config['data_dir']
         self._pending_restart = False
@@ -94,14 +94,12 @@ class Postgresql(object):
         self._schedule_load_slots = self.use_slots
 
         self._pgpass = config.get('pgpass') or os.path.join(os.path.expanduser('~'), 'pgpass')
-        self.callback = config.get('callbacks') or {}
+        self.__cb_called = False
         config_base_name = config.get('config_base_name', 'postgresql')
         self._postgresql_conf = os.path.join(self._data_dir, config_base_name + '.conf')
         self._postgresql_base_conf_name = config_base_name + '.base.conf'
         self._postgresql_base_conf = os.path.join(self._data_dir, self._postgresql_base_conf_name)
         self._recovery_conf = os.path.join(self._data_dir, 'recovery.conf')
-        self._configuration_to_save = (self._postgresql_conf, self._postgresql_base_conf,
-                                       os.path.join(self._data_dir, 'pg_hba.conf'))
         self._postmaster_pid = os.path.join(self._data_dir, 'postmaster.pid')
         self._trigger_file = config.get('recovery_conf', {}).get('trigger_file') or 'promote'
         self._trigger_file = os.path.abspath(os.path.join(self._data_dir, self._trigger_file))
@@ -123,10 +121,23 @@ class Postgresql(object):
             self.set_role('master' if self.is_leader() else 'replica')
             self._write_postgresql_conf()  # we are "joining" already running postgres
 
+    @property
+    def _configuration_to_save(self):
+        configuration = [self._postgresql_conf]
+        if 'custom_conf' not in self.config:
+            configuration.append(self._postgresql_base_conf)
+        if not self.config['parameters'].get('hba_file'):
+            configuration.append(os.path.join(self._data_dir, 'pg_hba.conf'))
+        return configuration
+
     @property
     def use_slots(self):
         return self._use_slots and self._major_version >= 9.4
 
+    @property
+    def callback(self):
+        return self.config.get('callbacks') or {}
+
     def _version_file_exists(self):
         return not self.data_directory_empty() and os.path.isfile(self._version_file)
 
@@ -147,15 +158,19 @@ class Postgresql(object):
 
     def resolve_connection_addresses(self):
         self._local_address = self.get_local_address()
-        self.connection_string = 'postgres://{connect_address}/{database}'.format(
+        self.connection_string = 'postgres://{0}/{1}'.format(
-            connect_address=self._connect_address or self._local_address, database=self._database)
+            self._connect_address or self._local_address['host'] + ':' + self._local_address['port'], self._database)
+
+    def _pgcommand(self, cmd):
+        """Returns path to the specified PostgreSQL command"""
+        return os.path.join(self._bin_dir, cmd)
 
     def pg_ctl(self, cmd, *args, **kwargs):
         """Builds and executes pg_ctl command
 
         :returns: `!True` when return_code == 0, otherwise `!False`"""
 
-        pg_ctl = ['pg_ctl', cmd]
+        pg_ctl = [self._pgcommand('pg_ctl'), cmd]
         if cmd in ('start', 'stop', 'restart'):
             pg_ctl += ['-w']
             timeout = self.config.get('pg_ctl_timeout')
@@ -240,7 +255,7 @@ class Postgresql(object):
         if not (self.config.get('use_pg_rewind') and all(self._superuser.get(n) for n in ('username', 'password'))):
             return False
 
-        cmd = ['pg_rewind', '--help']
+        cmd = [self._pgcommand('pg_rewind'), '--help']
         try:
             ret = subprocess.call(cmd, stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT)
             if ret != 0:  # pg_rewind is not there, close up the shop and go home
@@ -266,7 +281,7 @@ class Postgresql(object):
             if la.strip().lower() in ('*', '0.0.0.0', '127.0.0.1', 'localhost'):  # we are listening on '*' or localhost
                 local_address = 'localhost'  # connection via localhost is preferred
                 break
-        return local_address + ':' + self._server_parameters['port']
+        return {'host': local_address, 'port': self._server_parameters['port']}
 
     def get_postgres_role_from_data_directory(self):
         if self.data_directory_empty():
@@ -277,12 +292,21 @@ class Postgresql(object):
             return 'master'
 
     @property
-    def _connect_kwargs(self):
+    def _local_connect_kwargs(self):
-        return get_conn_kwargs('postgres://{0}/{1}'.format(self._local_address, self._database), self._superuser)
+        ret = self._local_address.copy()
+        ret.update({'database': self._database,
+                    'fallback_application_name': 'Patroni',
+                    'connect_timeout': 3,
+                    'options': '-c statement_timeout=2000'})
+        if 'username' in self._superuser:
+            ret['user'] = self._superuser['username']
+        if 'password' in self._superuser:
+            ret['password'] = self._superuser['password']
+        return ret
 
     def connection(self):
         if not self._connection or self._connection.closed != 0:
-            self._connection = psycopg2.connect(**self._connect_kwargs)
+            self._connection = psycopg2.connect(**self._local_connect_kwargs)
             self._connection.autocommit = True
             self.server_version = self._connection.server_version
         return self._connection
@@ -402,8 +426,8 @@ class Postgresql(object):
         # If there is no configuration key, or no value is specified, use basebackup
         replica_methods = self.config.get('create_replica_method') or ['basebackup']
 
-        if clone_member:
+        if clone_member and clone_member.conn_url:
-            r = get_conn_kwargs(clone_member.conn_url, self._replication)
+            r = clone_member.conn_kwargs(self._replication)
             connstring = 'postgres://{user}@{host}:{port}/{database}'.format(**r)
             # add the credentials to connect to the replica origin to pgpass.
             env = self.write_pgpass(r)
@@ -469,8 +493,15 @@ class Postgresql(object):
         except Exception:
             return False
 
+    @property
+    def cb_called(self):
+        return self.__cb_called
+
     def call_nowait(self, cb_name):
         """ pick a callback command and call it without waiting for it to finish """
+        if cb_name in (ACTION_ON_START, ACTION_ON_STOP, ACTION_ON_RESTART, ACTION_ON_ROLE_CHANGE):
+            self.__cb_called = True
+
         if not self.callback or cb_name not in self.callback:
             return False
         cmd = self.callback[cb_name]
@@ -500,6 +531,12 @@ class Postgresql(object):
             self._state = value
 
     def start(self, block_callbacks=False):
+        # make sure we close all connections established against
+        # the former node, otherwise, we might get a stalled one
+        # after kill -9, which would report incorrect data to
+        # patroni.
+        self.close_connection()
+
         if self.is_running():
             logger.error('Cannot start PostgreSQL because one is already running.')
             return True
@@ -538,7 +575,7 @@ class Postgresql(object):
 
     def checkpoint(self, connect_kwargs=None):
         check_not_is_in_recovery = connect_kwargs is not None
-        connect_kwargs = connect_kwargs or self._connect_kwargs
+        connect_kwargs = connect_kwargs or self._local_connect_kwargs
         for p in ['connect_timeout', 'options']:
             connect_kwargs.pop(p, None)
         try:
@@ -556,12 +593,6 @@ class Postgresql(object):
             return 'not accessible or not healty'
 
     def stop(self, mode='fast', block_callbacks=False, checkpoint=True):
-        # make sure we close all connections established against
-        # the former node, otherwise, we might get a stalled one
-        # after kill -9, which would report incorrect data to
-        # patroni.
-
-        self.close_connection()
         if not self.is_running():
             if not block_callbacks:
                 self.set_state('stopped')
@@ -601,12 +632,12 @@ class Postgresql(object):
 
     def _write_postgresql_conf(self):
         # rename the original configuration if it is necessary
-        if not os.path.exists(self._postgresql_base_conf):
+        if 'custom_conf' not in self.config and not os.path.exists(self._postgresql_base_conf):
             os.rename(self._postgresql_conf, self._postgresql_base_conf)
 
         with open(self._postgresql_conf, 'w') as f:
             f.write('# Do not edit this file manually!\n# It will be overwritten by Patroni!\n')
-            f.write("include '{0}'\n\n".format(self._postgresql_base_conf_name))
+            f.write("include '{0}'\n\n".format(self.config.get('custom_conf') or self._postgresql_base_conf_name))
             for name, value in sorted(self._server_parameters.items()):
                 if name not in self.CMDLINE_OPTIONS:
                     f.write("{0} = '{1}'\n".format(name, value))
@@ -624,31 +655,31 @@ class Postgresql(object):
         with open(os.path.join(self._data_dir, 'pg_hba.conf'), 'a') as f:
             f.write('\n{}\n'.format('\n'.join(config)))
 
-    def primary_conninfo(self, node_to_follow_url):
+    def primary_conninfo(self, member):
-        r = get_conn_kwargs(node_to_follow_url, self._replication)
+        if not (member and member.conn_url):
+            return None
+        r = member.conn_kwargs(self._replication)
         r.update({'application_name': self.name, 'sslmode': 'prefer', 'sslcompression': '1'})
         keywords = 'user password host port sslmode sslcompression application_name'.split()
         return ' '.join('{0}={{{0}}}'.format(kw) for kw in keywords).format(**r)
 
-    def check_recovery_conf(self, node_to_follow):
+    def check_recovery_conf(self, primary_conninfo):
         if not os.path.isfile(self._recovery_conf):
             return False
 
-        pattern = node_to_follow and node_to_follow.conn_url and self.primary_conninfo(node_to_follow.conn_url)
-
         with open(self._recovery_conf, 'r') as f:
             for line in f:
                 if line.startswith('primary_conninfo'):
-                    return pattern and (pattern in line)
+                    return primary_conninfo and (primary_conninfo in line)
-        return not pattern
+        return not primary_conninfo
 
-    def write_recovery_conf(self, node_to_follow):
+    def write_recovery_conf(self, primary_conninfo):
         with open(self._recovery_conf, 'w') as f:
             f.write("standby_mode = 'on'\nrecovery_target_timeline = 'latest'\n")
-            if node_to_follow and node_to_follow.conn_url:
+            if primary_conninfo:
-                f.write("primary_conninfo = '{0}'\n".format(self.primary_conninfo(node_to_follow.conn_url)))
+                f.write("primary_conninfo = '{0}'\n".format(primary_conninfo))
                 if self.use_slots:
-                    f.write("primary_slot_name = '{0}'\n".format(self.name))
+                    f.write("primary_slot_name = '{0}'\n".format(slot_name_from_member_name(self.name)))
             for name, value in self.config.get('recovery_conf', {}).items():
                 if name not in ('standby_mode', 'recovery_target_timeline', 'primary_conninfo', 'primary_slot_name'):
                     f.write("{0} = '{1}'\n".format(name, value))
@@ -659,7 +690,10 @@ class Postgresql(object):
         dsn = 'user={user} host={host} port={port} dbname={database} sslmode=prefer sslcompression=1'.format(**r)
         logger.info('running pg_rewind from %s', dsn)
         try:
-            return subprocess.call(['pg_rewind', '-D', self._data_dir, '--source-server', dsn], env=env) == 0
+            return subprocess.call([self._pgcommand('pg_rewind'),
+                                    '-D', self._data_dir,
+                                    '--source-server', dsn,
+                                    ], env=env) == 0
         except OSError:
             return False
 
@@ -669,7 +703,7 @@ class Postgresql(object):
         # Don't try to call pg_controldata during backup restore
         if self._version_file_exists() and self.state != 'creating replica':
             try:
-                data = subprocess.check_output(['pg_controldata', self._data_dir])
+                data = subprocess.check_output([self._pgcommand('pg_controldata'), self._data_dir])
                 if data:
                     data = data.decode('utf-8').splitlines()
                     result = {l.split(':')[0].replace('Current ', '', 1): l.split(':')[1].strip() for l in data if l}
@@ -695,7 +729,7 @@ class Postgresql(object):
 
     def single_user_mode(self, command=None, options=None):
         """ run a given command in a single-user mode. If the command is empty - then just start and stop """
-        cmd = ['postgres', '--single', '-D', self._data_dir]
+        cmd = [self._pgcommand('postgres'), '--single', '-D', self._data_dir]
         for opt, val in sorted((options or {}).items()):
             cmd.extend(['-c', '{0}={1}'.format(opt, val)])
         # need a database name to connect
@@ -723,37 +757,46 @@ class Postgresql(object):
         except OSError:
             logger.exception("Unable to list %s", status_dir)
 
-    def follow(self, member, leader, recovery=False):
+    @property
-        if self.check_recovery_conf(member) and not recovery:
+    def need_rewind(self):
+        return self._need_rewind
+
+    def follow(self, member, leader, recovery=False, async_executor=None, need_rewind=None):
+        if need_rewind is not None:
+            self._need_rewind = need_rewind
+
+        primary_conninfo = self.primary_conninfo(member)
+
+        if self.check_recovery_conf(primary_conninfo) and not recovery:
             return True
 
-        change_role = self.role == 'master'
+        if async_executor:
+            async_executor.schedule('changing primary_conninfo and restarting')
+            async_executor.run_async(self._do_follow, (primary_conninfo, leader, recovery))
+        else:
+            self._do_follow(primary_conninfo, leader, recovery)
 
-        if change_role:
+    def _do_follow(self, primary_conninfo, leader, recovery=False):
-            if leader:
+        change_role = self.role in ('master', 'demoted')
-                if leader.name == self.name:
+
-                    self._need_rewind = False
+        if leader and leader.name == self.name:
-                    member = None
+            primary_conninfo = None
-                    if self.is_running():
+            self._need_rewind = False
-                        return
+            if self.is_running():
-                else:
+                return
-                    self._need_rewind = bool(leader.conn_url) and self.can_rewind
+        elif change_role:
-            else:
+            self._need_rewind = True
-                self._need_rewind = False
+
-                member = None
+        self._need_rewind &= bool(leader and leader.conn_url) and self.can_rewind
 
         if self._need_rewind:
-            logger.info("set the rewind flag after demote")
+            logger.info("rewind flag is set")
 
-            self.set_role('unknown')
             if self.is_running() and not self.stop():
                 return logger.warning('Can not run pg_rewind because postgres is still running')
 
-            if not (leader and leader.conn_url):
-                return logger.info('Leader unknown, can not rewind')
-
             # prepare pg_rewind connection
-            r = get_conn_kwargs(leader.conn_url, self._superuser)
+            r = leader.conn_kwargs(self._superuser)
 
             # first make sure that we are really trying to rewind
             # from the master and run a checkpoint on a t in order to
@@ -779,17 +822,16 @@ class Postgresql(object):
             self.single_user_mode(options=opts)
 
             if self.rewind(r) or not self.config.get('remove_data_directory_on_rewind_failure', False):
-                self.write_recovery_conf(member)
+                self.write_recovery_conf(primary_conninfo)
                 ret = self.start()
             else:
                 logger.error('unable to rewind the former master')
                 self.remove_data_directory()
-                self.set_role('uninitialized')
                 ret = True
             self._need_rewind = False
         else:
-            self.write_recovery_conf(member)
+            self.write_recovery_conf(primary_conninfo)
-            ret = self.restart()
+            ret = self.start() if recovery else self.restart()
             self.set_role('replica')
 
         if change_role:
@@ -854,10 +896,17 @@ $$""".format(name, ' '.join(options)), name, password, password)
 
     def load_replication_slots(self):
         if self.use_slots and self._schedule_load_slots:
-            cursor = self.query("SELECT slot_name FROM pg_replication_slots WHERE slot_type='physical'")
+            cursor = self._query("SELECT slot_name FROM pg_replication_slots WHERE slot_type='physical'")
             self._replication_slots = [r[0] for r in cursor]
             self._schedule_load_slots = False
 
+    def postmaster_start_time(self):
+        try:
+            cursor = self.query("""SELECT to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ')""")
+            return cursor.fetchone()[0]
+        except psycopg2.Error:
+            return None
+
     def sync_replication_slots(self, cluster):
         if self.use_slots:
             try:
@@ -867,28 +916,40 @@ $$""".format(name, ' '.join(options)), name, password, password)
                 # the replicatefrom destination member is currently not a member of the cluster (fallback to the
                 # master), or if replicatefrom destination member happens to be the current master
                 if self.role == 'master':
-                    slots = [m.name for m in cluster.members if m.name != self.name and
+                    slot_members = [m.name for m in cluster.members if m.name != self.name and
-                             (m.replicatefrom is None or m.replicatefrom == self.name or
+                                    (m.replicatefrom is None or m.replicatefrom == self.name or
-                              not cluster.has_member(m.replicatefrom))]
+                                     not cluster.has_member(m.replicatefrom))]
                 else:
                     # only manage slots for replicas that replicate from this one, except for the leader among them
-                    slots = [m.name for m in cluster.members if m.replicatefrom == self.name and
+                    slot_members = [m.name for m in cluster.members if m.replicatefrom == self.name and
-                             m.name != cluster.leader.name]
+                                    m.name != cluster.leader.name]
+                slots = set(slot_name_from_member_name(name) for name in slot_members)
+
+                if len(slots) < len(slot_members):
+                    # Find which names are conflicting for a nicer error message
+                    slot_conflicts = defaultdict(list)
+                    for name in slot_members:
+                        slot_conflicts[slot_name_from_member_name(name)].append(name)
+                    logger.error("Following cluster members share a replication slot name: %s",
+                                 "; ".join("{} map to {}".format(", ".join(v), k)
+                                           for k, v in slot_conflicts.items() if len(v) > 1))
+
                 # drop unused slots
-                for slot in set(self._replication_slots) - set(slots):
+                for slot in set(self._replication_slots) - slots:
-                    self.query("""SELECT pg_drop_replication_slot(%s)
+                    self._query("""SELECT pg_drop_replication_slot(%s)
-                                   WHERE EXISTS(SELECT 1 FROM pg_replication_slots
+                                    WHERE EXISTS(SELECT 1 FROM pg_replication_slots
-                                   WHERE slot_name = %s)""", slot, slot)
+                                    WHERE slot_name = %s AND NOT active)""", slot, slot)
 
                 # create new slots
-                for slot in set(slots) - set(self._replication_slots):
+                for slot in slots - set(self._replication_slots):
-                    self.query("""SELECT pg_create_physical_replication_slot(%s)
+                    self._query("""SELECT pg_create_physical_replication_slot(%s)
-                                   WHERE NOT EXISTS (SELECT 1 FROM pg_replication_slots
+                                    WHERE NOT EXISTS (SELECT 1 FROM pg_replication_slots
-                                   WHERE slot_name = %s)""", slot, slot)
+                                    WHERE slot_name = %s)""", slot, slot)
 
                 self._replication_slots = slots
-            except psycopg2.Error:
+            except Exception:
                 logger.exception('Exception when changing replication slots')
+                self._schedule_load_slots = True
 
     def last_operation(self):
         return str(self.xlog_position())
@@ -928,6 +989,7 @@ $$""".format(name, ' '.join(options)), name, password, password)
                 logger.exception("Could not rename data directory %s", self._data_dir)
 
     def remove_data_directory(self):
+        self.set_role('uninitialized')
         logger.info('Removing data directory: %s', self._data_dir)
         try:
             if os.path.islink(self._data_dir):
@@ -951,7 +1013,7 @@ $$""".format(name, ' '.join(options)), name, password, password)
         ret = 1
         for bbfailures in range(0, maxfailures):
             try:
-                ret = subprocess.call(['pg_basebackup', '--pgdata=' + self._data_dir,
+                ret = subprocess.call([self._pgcommand('pg_basebackup'), '--pgdata=' + self._data_dir,
                                        '--xlog-method=stream', "--dbname=" + conn_url], env=env)
                 if ret == 0:
                     break
@@ -964,3 +1026,37 @@ $$""".format(name, ' '.join(options)), name, password, password)
                 time.sleep(5)
 
         return ret
+
+    @staticmethod
+    def postgres_version_to_int(pg_version):
+        """ Convert the server_version to integer
+
+        >>> Postgresql.postgres_version_to_int('9.5.3')
+        90503
+        >>> Postgresql.postgres_version_to_int('9.3.13')
+        90313
+        >>> Postgresql.postgres_version_to_int('10.1')
+        100001
+        >>> Postgresql.postgres_version_to_int('10')
+        Traceback (most recent call last):
+            ...
+        Exception: Invalid PostgreSQL format: X.Y or X.Y.Z is accepted: 10
+        >>> Postgresql.postgres_version_to_int('a.b.c')
+        Traceback (most recent call last):
+            ...
+        Exception: Invalid PostgreSQL version: a.b.c
+        """
+        components = pg_version.split('.')
+
+        result = []
+        if len(components) < 2 or len(components) > 3:
+            raise Exception("Invalid PostgreSQL format: X.Y or X.Y.Z is accepted: {0}".format(pg_version))
+        if len(components) == 2:
+            # new style verion numbers, i.e. 10.1 becomes 100001
+            components.insert(1, '0')
+        try:
+            result = [c if int(c) > 10 else '0{0}'.format(c) for c in components]
+            result = int(''.join(result))
+        except ValueError:
+            raise Exception("Invalid PostgreSQL version: {0}".format(pg_version))
+        return result

patroni/scripts/wale_restore.py

@@ -73,7 +73,7 @@ class WALERestore(object):
             # base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z
             # 20310671    00000001000000000000007F    00000040
             # 00000001000000000000007F    00000240
-            backup_strings = latest_backup.splitlines() if latest_backup else ()
+            backup_strings = latest_backup.decode('utf-8').splitlines() if latest_backup else ()
             if len(backup_strings) != 2:
                 return False
 

patroni/utils.py

@@ -2,6 +2,7 @@ import os
 import random
 import sys
 import time
+import re
 
 from patroni.exceptions import PatroniException
 
@@ -226,6 +227,10 @@ def reap_children():
             __reap_children = False
 
 
+def is_valid_pg_version(version):
+    return re.match(r'[1-9][0-9]?(\.(0|([1-9][0-9]?))){2}$', version)
+
+
 class RetryFailedError(PatroniException):
 
     """Raised when retrying an operation ultimately failed, after retrying the maximum number of attempts."""

patroni/version.py

@@ -1 +1 @@
-__version__ = '1.0'
+__version__ = '1.1'

postgres0.yml

@@ -4,10 +4,12 @@ name: postgresql0
 
 restapi:
   listen: 127.0.0.1:8008
+  connect_address: 127.0.0.1:8008
+#  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
+#  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
 #  authentication:
 #    username: username
 #    password: password
-  connect_address: 127.0.0.1:8008
 
 etcd:
   host: 127.0.0.1:4001
@@ -30,11 +32,11 @@ bootstrap:
 #        max_wal_senders: 5
 #        max_replication_slots: 5
 #        wal_log_hints: "on"
-        archive_mode: "on"
+#        archive_mode: "on"
-        archive_timeout: 1800s
+#        archive_timeout: 1800s
-        archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
+#        archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
-      recovery_conf:
+#      recovery_conf:
-        restore_command: cp ../wal_archive/%f %p
+#        restore_command: cp ../wal_archive/%f %p
 
   # some desired options for 'initdb'
   initdb:  # Note: It needs to be a list (some options need values, others are switches)
@@ -58,6 +60,7 @@ postgresql:
   listen: 127.0.0.1:5432
   connect_address: 127.0.0.1:5432
   data_dir: data/postgresql0
+#  bin_dir:
   pgpass: /tmp/pgpass0
   authentication:
     replication:

postgres1.yml

@@ -1,105 +1,77 @@
-ttl: &ttl 30
+scope: batman
-loop_wait: &loop_wait 10
+#namespace: /service/
-scope: &scope batman
+name: postgresql1
+
 restapi:
   listen: 127.0.0.1:8009
   connect_address: 127.0.0.1:8009
-#  auth: 'username:password'
 #  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
 #  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
+#  authentication:
+#    username: username
+#    password: password
+
 etcd:
-  scope: *scope
-  ttl: *ttl
   host: 127.0.0.1:4001
-  #discovery_srv: my-etcd.domain
+
-#consul:
+bootstrap:
-#  scope: *scope
+  # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
-#  ttl: *ttl
+  # and all other cluster members will use it as a `global configuration`
-#  host: 127.0.0.1:8500
+  dcs:
-#zookeeper:
+    ttl: 30
-#  scope: *scope
+    loop_wait: 10
-#  session_timeout: *ttl
+    retry_timeout: 10
-#  reconnect_timeout: *loop_wait
+    maximum_lag_on_failover: 1048576
-#  hosts:
+    postgresql:
-#    - 127.0.0.1:2181
+      use_pg_rewind: true
-#    - 127.0.0.2:2181
+#      use_slots: true
-#exhibitor:
+      parameters:
-#  poll_interval: 300
+#        wal_level: hot_standby
-#  port: 8181
+#        hot_standby: "on"
-#  hosts:
+#        wal_keep_segments: 8
-#    - host1
+#        max_wal_senders: 5
-#    - host2
+#        max_replication_slots: 5
-#    - host3
+#        wal_log_hints: "on"
+#        archive_mode: "on"
+#        archive_timeout: 1800s
+#        archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
+#      recovery_conf:
+#        restore_command: cp ../wal_archive/%f %p
+
+  # some desired options for 'initdb'
+  initdb:  # Note: It needs to be a list (some options need values, others are switches)
+  - encoding: UTF8
+  - data-checksums
+
+  pg_hba:  # Add following lines to pg_hba.conf after running 'initdb'
+  - host replication replicator 127.0.0.1/32 md5
+  - host all all 0.0.0.0/0 md5
+#  - hostssl all all 0.0.0.0/0 md5
+
+  # Some additional users users which needs to be created after initializing new cluster
+  users:
+    admin:
+      password: admin
+      options:
+        - createrole
+        - createdb
+
 postgresql:
-  name: postgresql1
-  scope: *scope
   listen: 127.0.0.1:5433
   connect_address: 127.0.0.1:5433
   data_dir: data/postgresql1
-  maximum_lag_on_failover: 1048576 # 1 megabyte in bytes
+#  bin_dir:
-  use_slots: True
   pgpass: /tmp/pgpass1
-  initdb:  ## We allow the following options to be passed on to initdb
+  authentication:
-  # - auth: authmethod
+    replication:
-  # - auth-host: authmethod
+      username: replicator
-  # - auth-local: authmethod
+      password: rep-pass
-  - encoding: UTF8
+    superuser:
-  # - data-checksums  # When pg_rewind is needed on 9.3, this needs to be enabled
+      username: postgres
-  # - locale: locale
+      password: zalando
-  # - lc-collate: locale
-  # - lc-ctype: locale
-  # - lc-messages: locale
-  # - lc-monetary: locale
-  # - lc-numeric: locale
-  # - lc-time: locale
-  # - text-search-config: CFG
-  # - xlogdir: directory
-  # - debug
-  # - noclean
-  pg_rewind:
-    username: postgres
-    password: zalando
-  pg_hba:
-  - host replication replicator 127.0.0.1/32 md5
-  - host all all 0.0.0.0/0 md5
-  # - hostssl all all 0.0.0.0/0 md5
-  replication:
-    username: replicator
-    password: rep-pass
-  superuser:
-    username: postgres
-    password: zalando
-  admin:
-    username: admin
-    password: admin
-# commented-out example for wal-e provisioning
-  create_replica_method:
-    - basebackup
-#   - wal_e
-# commented-out example for wal-e provisioning
-  #wal_e:
-    #command: /patroni/scripts/wale_restore.py
-    #env_dir: /home/postgres/etc/wal-e.d/env
-    #threshold_megabytes: 10240
-    #threshold_backup_size_percentage: 30
-    #retries: 2
-    #use_iam: 1
-  #recovery_conf:
-    #restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1
-  recovery_conf:
-    restore_command: cp ../wal_archive/%f %p
   parameters:
-    archive_mode: "on"
-    wal_level: hot_standby
-    archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
-    max_wal_senders: 10
-    wal_keep_segments: 8
-    archive_timeout: 1800s
-    max_replication_slots: 10
-    hot_standby: "on"
-    wal_log_hints: "on"
     unix_socket_directories: '.'
 tags:
-    nofailover: False
+    nofailover: false
-    noloadbalance: False
+    noloadbalance: false
-    clonefrom: False
+    clonefrom: false

postgres2.yml

@@ -1,106 +1,78 @@
-ttl: &ttl 30
+scope: batman
-loop_wait: &loop_wait 10
+#namespace: /service/
-scope: &scope batman
+name: postgresql2
+
 restapi:
   listen: 127.0.0.1:8010
   connect_address: 127.0.0.1:8010
-  auth: 'username:password'
 #  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
 #  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
+  authentication:
+    username: username
+    password: password
+
 etcd:
-  scope: *scope
-  ttl: *ttl
   host: 127.0.0.1:4001
-  #discovery_srv: my-etcd.domain
+
-#consul:
+bootstrap:
-#  scope: *scope
+  # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
-#  ttl: *ttl
+  # and all other cluster members will use it as a `global configuration`
-#  host: 127.0.0.1:8500
+  dcs:
-#zookeeper:
+    ttl: 30
-#  scope: *scope
+    loop_wait: 10
-#  session_timeout: *ttl
+    retry_timeout: 10
-#  reconnect_timeout: *loop_wait
+    maximum_lag_on_failover: 1048576
-#  hosts:
+    postgresql:
-#    - 127.0.0.1:2181
+      use_pg_rewind: true
-#    - 127.0.0.2:2181
+#      use_slots: true
-#exhibitor:
+      parameters:
-#  poll_interval: 300
+#        wal_level: hot_standby
-#  port: 8181
+#        hot_standby: "on"
-#  hosts:
+#        wal_keep_segments: 8
-#    - host1
+#        max_wal_senders: 5
-#    - host2
+#        max_replication_slots: 5
-#    - host3
+#        wal_log_hints: "on"
+#        archive_mode: "on"
+#        archive_timeout: 1800s
+#        archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
+#      recovery_conf:
+#        restore_command: cp ../wal_archive/%f %p
+
+  # some desired options for 'initdb'
+  initdb:  # Note: It needs to be a list (some options need values, others are switches)
+  - encoding: UTF8
+  - data-checksums
+
+  pg_hba:  # Add following lines to pg_hba.conf after running 'initdb'
+  - host replication replicator 127.0.0.1/32 md5
+  - host all all 0.0.0.0/0 md5
+#  - hostssl all all 0.0.0.0/0 md5
+
+  # Some additional users users which needs to be created after initializing new cluster
+  users:
+    admin:
+      password: admin
+      options:
+        - createrole
+        - createdb
+
 postgresql:
-  name: postgresql2
-  scope: *scope
   listen: 127.0.0.1:5434
   connect_address: 127.0.0.1:5434
   data_dir: data/postgresql2
-  maximum_lag_on_failover: 1048576 # 1 megabyte in bytes
+#  bin_dir:
-  use_slots: True
   pgpass: /tmp/pgpass2
-  initdb:  ## We allow the following options to be passed on to initdb
+  authentication:
-  # - auth: authmethod
+    replication:
-  # - auth-host: authmethod
+      username: replicator
-  # - auth-local: authmethod
+      password: rep-pass
-  - encoding: UTF8
+    superuser:
-  # - data-checksums  # When pg_rewind is needed on 9.3, this needs to be enabled
+      username: postgres
-  # - locale: locale
+      password: zalando
-  # - lc-collate: locale
-  # - lc-ctype: locale
-  # - lc-messages: locale
-  # - lc-monetary: locale
-  # - lc-numeric: locale
-  # - lc-time: locale
-  # - text-search-config: CFG
-  # - xlogdir: directory
-  # - debug
-  # - noclean
-  pg_rewind:
-    username: postgres
-    password: zalando
-  pg_hba:
-  - host replication replicator 127.0.0.1/32 md5
-  - host all all 0.0.0.0/0 md5
-  # - hostssl all all 0.0.0.0/0 md5
-  replication:
-    username: replicator
-    password: rep-pass
-  superuser:
-    username: postgres
-    password: zalando
-  admin:
-    username: admin
-    password: admin
-# commented-out example for wal-e provisioning
-  create_replica_method:
-    - basebackup
-#   - wal_e
-# commented-out example for wal-e provisioning
-  #wal_e:
-    #command: /patroni/scripts/wale_restore.py
-    #env_dir: /home/postgres/etc/wal-e.d/env
-    #threshold_megabytes: 10240
-    #threshold_backup_size_percentage: 30
-    #retries: 2
-    #use_iam: 1
-  #recovery_conf:
-    #restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1
-  recovery_conf:
-    restore_command: cp ../wal_archive/%f %p
   parameters:
-    archive_mode: "on"
-    wal_level: hot_standby
-    archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
-    max_wal_senders: 10
-    wal_keep_segments: 8
-    archive_timeout: 1800s
-    max_replication_slots: 10
-    hot_standby: "on"
-    wal_log_hints: "on"
     unix_socket_directories: '.'
 tags:
-    nofailover: False
+    nofailover: false
-    noloadbalance: False
+    noloadbalance: false
-    clonefrom: False
+    clonefrom: false
-    replicatefrom: postgresql1
+    replicatefrom: postgres1

tests/test_api.py

@@ -1,5 +1,7 @@
+import datetime
 import json
 import psycopg2
+import pytz
 import unittest
 
 from mock import Mock, patch
@@ -10,6 +12,10 @@ from six.moves import BaseHTTPServer
 from test_postgresql import psycopg2_connect, MockCursor
 
 
+future_restart_time = datetime.datetime.now(pytz.utc) + datetime.timedelta(days=5)
+postmaster_start_time = datetime.datetime.now(pytz.utc)
+
+
 class MockPostgresql(object):
 
     name = 'test'
@@ -24,14 +30,17 @@ class MockPostgresql(object):
     def connection():
         return psycopg2_connect()
 
+    @staticmethod
+    def postmaster_start_time():
+        return str(postmaster_start_time)
+
 
 class MockHa(object):
 
-    dcs = Mock()
     state_handler = MockPostgresql()
 
     @staticmethod
-    def schedule_reinitialize():
+    def reinitialize():
         return 'reinitialize'
 
     @staticmethod
@@ -42,21 +51,30 @@ class MockHa(object):
     def restart_scheduled():
         return False
 
+    @staticmethod
+    def delete_future_restart():
+        return True
+
     @staticmethod
     def fetch_nodes_statuses(members):
         return [[None, True, None, None, {}]]
 
+    @staticmethod
+    def schedule_future_restart(data):
+        return True
+
 
 class MockPatroni(object):
 
-    nap_time = 10
-    config = Mock()
-    postgresql = MockPostgresql()
     ha = MockHa()
+    config = Mock()
+    postgresql = ha.state_handler
     dcs = Mock()
     tags = {}
     version = '0.00'
     noloadbalance = Mock(return_value=False)
+    scheduled_restart = {'schedule': future_restart_time,
+                         'postmaster_start_time': postgresql.postmaster_start_time()}
 
     @staticmethod
     def sighup_handler():
@@ -118,14 +136,14 @@ class TestRestApiHandler(unittest.TestCase):
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0'))
         MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0\nAuthorization:')
 
-    @patch.object(MockHa, 'dcs')
+    @patch.object(MockPatroni, 'dcs')
     def test_do_GET_config(self, mock_dcs):
         mock_dcs.cluster.config.data = {}
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /config'))
         mock_dcs.cluster.config = None
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /config'))
 
-    @patch.object(MockHa, 'dcs')
+    @patch.object(MockPatroni, 'dcs')
     def test_do_PATCH_config(self, mock_dcs):
         config = {'postgresql': {'use_slots': False, 'use_pg_rewind': True, 'parameters': {'wal_level': 'logical'}}}
         mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, json.dumps(config))
@@ -141,7 +159,7 @@ class TestRestApiHandler(unittest.TestCase):
         mock_dcs.set_config_value.return_value = False
         MockRestApiServer(RestApiHandler, request)
 
-    @patch.object(MockHa, 'dcs')
+    @patch.object(MockPatroni, 'dcs')
     def test_do_PUT_config(self, mock_dcs):
         mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, '{}')
         request = 'PUT /config HTTP/1.0' + self._authorization + '\nContent-Length: '
@@ -161,23 +179,67 @@ class TestRestApiHandler(unittest.TestCase):
             MockRestApiServer(RestApiHandler, 'POST /reload HTTP/1.0' + self._authorization)
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /reload HTTP/1.0' + self._authorization))
 
-    def test_do_POST_restart(self):
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_POST_restart(self, mock_dcs):
+        mock_dcs.get_cluster.return_value.is_paused.return_value = False
         request = 'POST /restart HTTP/1.0' + self._authorization
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
+
         with patch.object(MockHa, 'restart', Mock(side_effect=Exception)):
             MockRestApiServer(RestApiHandler, request)
 
-    @patch.object(MockHa, 'dcs')
+        post = request + '\nContent-Length: '
-    def test_do_POST_reinitialize(self, dcs):
+
-        cluster = dcs.get_cluster.return_value
+        def make_request(request=None, **kwargs):
+            request = json.dumps(kwargs) if request is None else request
+            return '{0}{1}\n\n{2}'.format(post, len(request), request)
+
+        # empty request
+        request = make_request('')
+        MockRestApiServer(RestApiHandler, request)
+        # invalid request
+        request = make_request('foobar=baz')
+        MockRestApiServer(RestApiHandler, request)
+        # wrong role
+        request = make_request(schedule=future_restart_time.isoformat(), role='unknown', postgres_version='9.5.3')
+        MockRestApiServer(RestApiHandler, request)
+        # wrong version
+        request = make_request(schedule=future_restart_time.isoformat(), role='master', postgres_version='9.5.3.1')
+        MockRestApiServer(RestApiHandler, request)
+        # unknown filter
+        request = make_request(schedule=future_restart_time.isoformat(), batman='lives')
+        MockRestApiServer(RestApiHandler, request)
+        # incorrect schedule
+        request = make_request(schedule='2016-08-42 12:45TZ+1', role='master')
+        MockRestApiServer(RestApiHandler, request)
+        # everything fine, but the schedule is missing
+        request = make_request(role='master', postgres_version='9.5.2')
+        MockRestApiServer(RestApiHandler, request)
+        for retval in (True, False):
+            with patch.object(MockHa, 'schedule_future_restart', Mock(return_value=retval)):
+                request = make_request(schedule=future_restart_time.isoformat())
+                MockRestApiServer(RestApiHandler, request)
+            with patch.object(MockHa, 'restart', Mock(return_value=(retval, "foo"))):
+                request = make_request(role='master', postgres_version='9.5.2')
+                MockRestApiServer(RestApiHandler, request)
+
+        mock_dcs.get_cluster.return_value.is_paused.return_value = True
+        MockRestApiServer(RestApiHandler, make_request(schedule='2016-08-42 12:45TZ+1', role='master'))
+
+    def test_do_DELETE_restart(self):
+        for retval in (True, False):
+            with patch.object(MockHa, 'delete_future_restart', Mock(return_value=retval)):
+                request = 'DELETE /restart HTTP/1.0' + self._authorization
+                self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
+
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_POST_reinitialize(self, mock_dcs):
+        cluster = mock_dcs.get_cluster.return_value
+        cluster.is_paused.return_value = False
         request = 'POST /reinitialize HTTP/1.0' + self._authorization
         MockRestApiServer(RestApiHandler, request)
-        cluster.is_unlocked.return_value = False
+        with patch.object(MockHa, 'reinitialize', Mock(return_value=None)):
-        MockRestApiServer(RestApiHandler, request)
-        with patch.object(MockHa, 'schedule_reinitialize', Mock(return_value=None)):
             MockRestApiServer(RestApiHandler, request)
-        cluster.leader.name = 'test'
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
 
     @patch('time.sleep', Mock())
     def test_RestApiServer_query(self):
@@ -187,8 +249,9 @@ class TestRestApiHandler(unittest.TestCase):
             self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
 
     @patch('time.sleep', Mock())
-    @patch.object(MockHa, 'dcs')
+    @patch.object(MockPatroni, 'dcs')
     def test_do_POST_failover(self, dcs):
+        dcs.loop_wait = 10
         cluster = dcs.get_cluster.return_value
 
         post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
@@ -213,19 +276,27 @@ class TestRestApiHandler(unittest.TestCase):
         cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
                            Member(0, 'postgresql2', 30, {'api_url': 'http'})]
         MockRestApiServer(RestApiHandler, request)
-        with patch.object(MockPatroni, 'dcs') as d:
+
-            cluster = d.get_cluster.return_value
+        cluster.failover = None
-            cluster.leader.name = 'postgresql0'
+        MockRestApiServer(RestApiHandler, request)
-            MockRestApiServer(RestApiHandler, request)
+
-            cluster.leader.name = 'postgresql2'
+        dcs.get_cluster.side_effect = [cluster]
-            MockRestApiServer(RestApiHandler, request)
+        MockRestApiServer(RestApiHandler, request)
-            cluster.leader.name = 'postgresql1'
+
-            cluster.failover = None
+        cluster2 = cluster.copy()
-            MockRestApiServer(RestApiHandler, request)
+        cluster2.leader.name = 'postgresql0'
-            d.get_cluster = Mock(side_effect=Exception)
+        dcs.get_cluster.side_effect = [cluster, cluster2]
-            MockRestApiServer(RestApiHandler, request)
+        MockRestApiServer(RestApiHandler, request)
-            d.manual_failover.return_value = False
+
-            MockRestApiServer(RestApiHandler, request)
+        cluster2.leader.name = 'postgresql2'
+        dcs.get_cluster.side_effect = [cluster, cluster2]
+        MockRestApiServer(RestApiHandler, request)
+
+        dcs.get_cluster.side_effect = None
+        dcs.manual_failover.return_value = False
+        MockRestApiServer(RestApiHandler, request)
+        dcs.manual_failover.return_value = True
+
         with patch.object(MockHa, 'fetch_nodes_statuses', Mock(return_value=[])):
             MockRestApiServer(RestApiHandler, request)
 

tests/test_ctl.py

@@ -6,12 +6,13 @@ import unittest
 
 from click.testing import CliRunner
 from mock import patch, Mock
-from patroni.ctl import ctl, members, store_config, load_config, output_members, post_patroni, get_dcs, parse_dcs, \
+from patroni.ctl import ctl, members, store_config, load_config, output_members, request_patroni, get_dcs, parse_dcs, \
     wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException
+from patroni.dcs.etcd import Client
 from psycopg2 import OperationalError
 from test_etcd import etcd_read, requests_get, socket_getaddrinfo, MockResponse
 from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \
-    get_cluster_initialized_with_only_leader
+    get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader
 from test_postgresql import MockConnect, psycopg2_connect
 
 CONFIG_FILE_PATH = './test-ctl.yaml'
@@ -28,14 +29,16 @@ def test_rw_config():
         os.rmdir(CONFIG_FILE_PATH)
 
 
-@patch('patroni.ctl.load_config', Mock(return_value={'restapi': {'auth': 'u:p'}, 'etcd': {'host': 'localhost:4001'}}))
+@patch('patroni.ctl.load_config',
+       Mock(return_value={'postgresql': {'data_dir': '.', 'parameters': {}, 'retry_timeout': 5},
+                          'restapi': {'auth': 'u:p', 'listen': ''}, 'etcd': {'host': 'localhost:4001'}}))
 class TestCtl(unittest.TestCase):
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
     def setUp(self):
-        self.runner = CliRunner()
+        with patch.object(Client, 'machines') as mock_machines:
-        with patch.object(etcd.Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
+            self.runner = CliRunner()
             self.e = get_dcs({'etcd': {'ttl': 30, 'host': 'ok:2379', 'retry_timeout': 10}}, 'foo')
 
     @patch('psycopg2.connect', psycopg2_connect)
@@ -66,7 +69,7 @@ class TestCtl(unittest.TestCase):
         self.assertIsNone(output_members(cluster, name='abc', fmt='tsv'))
 
     @patch('patroni.ctl.get_dcs')
-    @patch('patroni.ctl.post_patroni', Mock(return_value=MockResponse()))
+    @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
     def test_failover(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
@@ -79,6 +82,11 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2030-01-01T12:23:00\ny')
         assert result.exit_code == 0
 
+        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            result = self.runner.invoke(ctl,
+                                        ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
+            assert result.exit_code == 1
+
         # Aborting failover,as we anser NO to the confirmation
         result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\nN')
         assert result.exit_code == 1
@@ -109,12 +117,12 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['failover', 'dummy'], input='dummy')
         assert result.exit_code == 1
 
-        with patch('patroni.ctl.post_patroni', Mock(side_effect=Exception)):
+        with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)):
             # Non-responding patroni
             result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
             assert 'falling back to DCS' in result.output
 
-        with patch('patroni.ctl.post_patroni') as mocked:
+        with patch('patroni.ctl.request_patroni') as mocked:
             mocked.return_value.status_code = 500
             result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
             assert 'Failover failed' in result.output
@@ -205,23 +213,79 @@ class TestCtl(unittest.TestCase):
     @patch('patroni.ctl.get_dcs')
     def test_restart_reinit(self, mock_get_dcs):
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
-        result = self.runner.invoke(ctl, ['restart', 'alpha'], input='y')
+        result = self.runner.invoke(ctl, ['restart', 'alpha'], input='y\n\nnow')
-        assert 'restart failed for' in result.output
+        assert 'Failed: restart for' in result.output
         assert result.exit_code == 0
 
         result = self.runner.invoke(ctl, ['reinit', 'alpha'], input='y')
         assert result.exit_code == 1
 
+        # successful reinit
+        result = self.runner.invoke(ctl, ['reinit', 'alpha', 'other'], input='y')
+        assert result.exit_code == 0
+
         # Aborted restart
         result = self.runner.invoke(ctl, ['restart', 'alpha'], input='N')
         assert result.exit_code == 1
 
+        result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force'])
+        assert result.exit_code == 0
+
         # Not a member
         result = self.runner.invoke(ctl, ['restart', 'alpha', 'dummy', '--any'], input='y')
         assert result.exit_code == 1
 
+        # Wrong pg version
+        result = self.runner.invoke(ctl, ['restart', 'alpha', '--any', '--pg-version', '9.1'], input='y')
+        assert 'Error: PostgreSQL version' in result.output
+        assert result.exit_code == 1
+
+        with patch('requests.delete', Mock(return_value=MockResponse(500))):
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force',
+                                              '--scheduled', '2300-10-01T14:30'])
+            assert 'Failed: flush scheduled restart' in result.output
+
+        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            result = self.runner.invoke(ctl,
+                                        ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
+            assert result.exit_code == 1
+
         with patch('requests.post', Mock(return_value=MockResponse())):
-            result = self.runner.invoke(ctl, ['restart', 'alpha'], input='y')
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0.0',
+                                        '--scheduled', '2300-10-01T14:30'], input='y')
+            assert result.exit_code == 0
+
+        with patch('requests.post', Mock(return_value=MockResponse(204))):
+            # get restart with the non-200 return code
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0.0',
+                                        '--scheduled', '2300-10-01T14:30'], input='y')
+            assert result.exit_code == 0
+
+        # force restart with restart already present
+        with patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse(204))):
+            result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force',
+                                              '--scheduled', '2300-10-01T14:30'])
+            assert result.exit_code == 0
+
+        with patch('requests.post', Mock(return_value=MockResponse(202))):
+            # get restart with the non-200 return code
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(
+                ctl, ['restart', 'alpha', '--pg-version', '99.0.0', '--scheduled', '2300-10-01T14:30'], input='y'
+            )
+            assert 'Success: restart scheduled' in result.output
+            assert result.exit_code == 0
+
+        with patch('requests.post', Mock(return_value=MockResponse(409))):
+            # get restart with the non-200 return code
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(
+                ctl, ['restart', 'alpha', '--pg-version', '99.0.0', '--scheduled', '2300-10-01T14:30'], input='y'
+            )
+            assert 'Failed: another restart is already' in result.output
             assert result.exit_code == 0
 
     @patch('patroni.ctl.get_dcs')
@@ -253,9 +317,9 @@ class TestCtl(unittest.TestCase):
         assert cluster.leader.member.name == 'leader'
 
     @patch('requests.post', Mock(side_effect=requests.exceptions.ConnectionError('foo')))
-    def test_post_patroni(self):
+    def test_request_patroni(self):
         member = get_cluster_initialized_with_leader().leader.member
-        self.assertRaises(requests.exceptions.ConnectionError, post_patroni, member, 'dummy', {})
+        self.assertRaises(requests.exceptions.ConnectionError, request_patroni, member, 'post', 'dummy', {})
 
     def test_ctl(self):
         self.runner.invoke(ctl, ['list'])
@@ -292,3 +356,87 @@ class TestCtl(unittest.TestCase):
     def test_configure(self):
         result = self.runner.invoke(configure, ['--dcs', 'abc', '-c', 'dummy', '-n', 'bla'])
         assert result.exit_code == 0
+
+    @patch('patroni.ctl.get_dcs')
+    def test_scaffold(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_not_initialized_without_leader
+        mock_get_dcs.return_value.initialize = Mock(return_value=True)
+        mock_get_dcs.return_value.touch_member = Mock(return_value=True)
+        mock_get_dcs.return_value.attempt_to_acquire_leader = Mock(return_value=True)
+
+        with patch.object(self.e, 'initialize', return_value=False):
+            result = self.runner.invoke(ctl, ['scaffold', 'alpha'])
+            assert result.exception
+
+        with patch.object(mock_get_dcs.return_value, 'touch_member', Mock(return_value=False)):
+            result = self.runner.invoke(ctl, ['scaffold', 'alpha'])
+            assert result.exception
+
+        result = self.runner.invoke(ctl, ['scaffold', 'alpha'])
+        assert result.exit_code == 0
+
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        result = self.runner.invoke(ctl, ['scaffold', 'alpha'])
+        assert result.exception
+
+    @patch('patroni.ctl.get_dcs')
+    def test_list_extended(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+
+        result = self.runner.invoke(ctl, ['list', 'dummy', '--extended'])
+        assert '2100' in result.output
+        assert 'Scheduled restart' in result.output
+
+    @patch('patroni.ctl.get_dcs')
+    @patch('requests.delete', Mock(return_value=MockResponse()))
+    def test_flush(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+
+        result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '-r', 'master'], input='y')
+        assert 'No scheduled restart' in result.output
+
+        result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force'])
+        assert 'Success: flush scheduled restart' in result.output
+        with patch.object(requests, 'delete', return_value=MockResponse(404)):
+            result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force'])
+            assert 'Failed: flush scheduled restart' in result.output
+
+    @patch('patroni.ctl.get_dcs')
+    def test_pause_cluster(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+
+        with patch('requests.patch', Mock(return_value=MockResponse(200))):
+            result = self.runner.invoke(ctl, ['pause', 'dummy'])
+            assert 'Success' in result.output
+
+        with patch('requests.patch', Mock(return_value=MockResponse(500))):
+            result = self.runner.invoke(ctl, ['pause', 'dummy'])
+            assert 'Failed' in result.output
+
+        with patch('requests.patch', Mock(return_value=MockResponse(200))),\
+                patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            result = self.runner.invoke(ctl, ['pause', 'dummy'])
+            assert 'Cluster is already paused' in result.output
+
+    @patch('patroni.ctl.get_dcs')
+    def test_resume_cluster(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+
+        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            with patch('requests.patch', Mock(return_value=MockResponse(200))):
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Success' in result.output
+
+            with patch('requests.patch', Mock(return_value=MockResponse(500))):
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Failed' in result.output
+
+            with patch('requests.patch', Mock(return_value=MockResponse(200))),\
+                    patch('patroni.dcs.Cluster.is_paused', Mock(return_value=False)):
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Cluster is not paused' in result.output

tests/test_etcd.py

@@ -13,8 +13,8 @@ from urllib3.exceptions import ReadTimeoutError
 
 class MockResponse(object):
 
-    def __init__(self):
+    def __init__(self, status_code=200):
-        self.status_code = 200
+        self.status_code = status_code
         self.content = '{}'
         self.ok = True
         self.text = ''
@@ -103,7 +103,9 @@ def etcd_read(self, key, **kwargs):
                      "expiration": "2015-05-15T09:11:09.611860899Z", "ttl": 30,
                      "modifiedIndex": 20730, "createdIndex": 20730}],
                  "modifiedIndex": 1581, "createdIndex": 1581}], "modifiedIndex": 1581, "createdIndex": 1581}}
-    return etcd.EtcdResult(**response)
+    result = etcd.EtcdResult(**response)
+    result.etcd_index = 0
+    return result
 
 
 class SleepException(Exception):
@@ -132,6 +134,10 @@ def socket_getaddrinfo(*args):
 def http_request(method, url, **kwargs):
     if url == 'http://localhost:2379/timeout':
         raise ReadTimeoutError(None, None, None)
+    if url == 'http://localhost:2379/v2/machines':
+        ret = MockResponse()
+        ret.content = 'http://localhost:2379,http://localhost:4001'
+        return ret
     if url == 'http://localhost:2379/':
         return MockResponse()
     raise socket.error
@@ -145,26 +151,39 @@ class TestClient(unittest.TestCase):
     @patch('dns.resolver.query', dns_query)
     @patch('requests.get', requests_get)
     def setUp(self):
-        with patch.object(etcd.Client, 'machines') as mock_machines:
+        with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
             self.client = Client({'discovery_srv': 'test', 'retry_timeout': 3})
             self.client.http.request = http_request
             self.client.http.request_encode_body = http_request
 
-    def test_api_execute(self):
+    def test_machines(self):
         self.client._base_uri = 'http://localhost:4001'
         self.client._machines_cache = ['http://localhost:2379']
-        self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
+        self.assertIsNotNone(self.client.machines)
-        self.client._update_machines_cache = False
-        self.client.api_execute('/', 'POST', timeout=0)
-        self.client._update_machines_cache = False
         self.client._base_uri = 'http://localhost:4001'
         self.client._machines_cache = []
-        self.assertRaises(etcd.EtcdConnectionFailed, self.client.api_execute, '/', 'GET')
+        self.assertIsNotNone(self.client.machines)
-        self.assertTrue(self.client._update_machines_cache)
+        self.client._update_machines_cache = True
-        self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'GET')
+        machines = None
-        self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
+        try:
+            machines = self.client.machines
+            self.assertFail()
+        except Exception:
+            self.assertIsNone(machines)
+
+    @patch.object(Client, 'machines')
+    def test_api_execute(self, mock_machines):
+        mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
         self.assertRaises(ValueError, self.client.api_execute, '', '')
+        self.client._base_uri = 'http://localhost:4001'
+        self.client._machines_cache = ['http://localhost:2379']
+        self.client.api_execute('/', 'POST', timeout=0)
+        self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
+        self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
+        self.client._update_machines_cache = True
+        with patch.object(Client, '_load_machines_cache', Mock(side_effect=etcd.EtcdException)):
+            self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'GET')
 
     def test_get_srv_record(self):
         self.assertEquals(self.client.get_srv_record('blabla'), [])
@@ -177,7 +196,9 @@ class TestClient(unittest.TestCase):
     def test__get_machines_cache_from_dns(self):
         self.client._get_machines_cache_from_dns('error:2379')
 
-    def test__load_machines_cache(self):
+    @patch.object(Client, 'machines')
+    def test__load_machines_cache(self, mock_machines):
+        mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
         self.client._config = {}
         self.assertRaises(Exception, self.client._load_machines_cache)
         self.client._config = {'discovery_srv': 'blabla'}
@@ -201,9 +222,9 @@ class TestEtcd(unittest.TestCase):
 
     @patch('dns.resolver.query', dns_query)
     def test_get_etcd_client(self):
-        with patch.object(etcd.Client, 'machines') as mock_machines:
+        with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(side_effect=etcd.EtcdException)
-            with patch('time.sleep', Mock(side_effect=SleepException())):
+            with patch('time.sleep', Mock(side_effect=SleepException)):
                 self.assertRaises(SleepException, self.etcd.get_etcd_client,
                                   {'discovery_srv': 'test', 'retry_timeout': 10})
 

tests/test_ha.py

@@ -4,9 +4,10 @@ import os
 import pytz
 import unittest
 
-from mock import Mock, MagicMock, patch
+from mock import Mock, MagicMock, PropertyMock, patch
 from patroni.config import Config
 from patroni.dcs import Cluster, Failover, Leader, Member, get_dcs
+from patroni.dcs.etcd import Client
 from patroni.exceptions import DCSError, PostgresException
 from patroni.ha import Ha
 from patroni.postgresql import Postgresql
@@ -34,7 +35,10 @@ def get_cluster_initialized_without_leader(leader=False, failover=None):
                                   'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location': 4})
     l = Leader(0, 0, m1) if leader else None
     m2 = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
-                                 'api_url': 'http://127.0.0.1:8011/patroni', 'tags': {'clonefrom': True}})
+                                 'api_url': 'http://127.0.0.1:8011/patroni',
+                                 'tags': {'clonefrom': True},
+                                 'scheduled_restart': {'schedule': "2100-01-01 10:53:07.560445+00:00",
+                                                       'postgres_version': '99.0.0'}})
     return get_cluster(True, l, [m1, m2], failover)
 
 
@@ -46,6 +50,9 @@ def get_cluster_initialized_with_only_leader(failover=None):
     l = get_cluster_initialized_without_leader(leader=True, failover=failover).leader
     return get_cluster(True, l, [l], failover)
 
+future_restart_time = datetime.datetime.now(pytz.utc) + datetime.timedelta(days=5)
+postmaster_start_time = datetime.datetime.now(pytz.utc)
+
 
 class MockPatroni(object):
 
@@ -76,13 +83,14 @@ zookeeper:
         self.api = Mock()
         self.tags = {'foo': 'bar'}
         self.nofailover = None
-        self.nap_time = 10
         self.replicatefrom = None
         self.api.connection_string = 'http://127.0.0.1:8008'
         self.clonefrom = None
+        self.scheduled_restart = {'schedule': future_restart_time,
+                                  'postmaster_start_time': str(postmaster_start_time)}
 
 
-def run_async(func, args=()):
+def run_async(self, func, args=()):
     return func(*args) if args else func()
 
 
@@ -101,13 +109,15 @@ def run_async(func, args=()):
 @patch.object(etcd.Client, 'write', etcd_write)
 @patch.object(etcd.Client, 'read', etcd_read)
 @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
+@patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=False))
+@patch('patroni.async_executor.AsyncExecutor.run_async', run_async)
 @patch('subprocess.call', Mock(return_value=0))
 class TestHa(unittest.TestCase):
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
     @patch.object(etcd.Client, 'read', etcd_read)
     def setUp(self):
-        with patch.object(etcd.Client, 'machines') as mock_machines:
+        with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
             self.p = Postgresql({'name': 'postgresql0', 'scope': 'dummy', 'listen': '127.0.0.1:5432',
                                  'data_dir': 'data/postgresql0', 'retry_timeout': 10,
@@ -117,12 +127,12 @@ class TestHa(unittest.TestCase):
                                                 'hot_standby': 'on', 'max_wal_senders': 5, 'wal_keep_segments': 8}})
             self.p.set_state('running')
             self.p.set_role('replica')
+            self.p.postmaster_start_time = MagicMock(return_value=str(postmaster_start_time))
             self.p.check_replication_lag = true
             self.p.can_create_replica_without_replication_connection = MagicMock(return_value=False)
             self.e = get_dcs({'etcd': {'ttl': 30, 'host': 'ok:2379', 'scope': 'test',
                                        'name': 'foo', 'retry_timeout': 10}})
             self.ha = Ha(MockPatroni(self.p, self.e))
-            self.ha._async_executor.run_async = run_async
             self.ha.old_cluster = self.e.get_cluster()
             self.ha.cluster = get_cluster_not_initialized_without_leader()
             self.ha.load_cluster_from_dcs = Mock()
@@ -157,6 +167,9 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.assertEquals(self.ha.run_cycle(), 'starting as readonly because i had the session lock')
 
+    def test_do_not_recover_in_pause(self):
+        pass
+
     @patch('sys.exit', return_value=1)
     @patch('patroni.ha.Ha.sysid_valid', MagicMock(return_value=True))
     def test_sysid_no_match(self, exit_mock):
@@ -225,6 +238,13 @@ class TestHa(unittest.TestCase):
         self.ha.patroni.replicatefrom = "foo"
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am a secondary and i am following a leader')
 
+    def test_follow_in_pause(self):
+        self.ha.cluster.is_unlocked = false
+        self.ha.is_paused = true
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: continue to run as master without lock')
+        self.p.is_leader = false
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: no action')
+
     def test_no_etcd_connection_master_demote(self):
         self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
         self.assertEquals(self.ha.run_cycle(), 'demoted self because DCS is not accessible and i was a leader')
@@ -258,41 +278,40 @@ class TestHa(unittest.TestCase):
         self.assertRaises(PostgresException, self.ha.bootstrap)
 
     def test_reinitialize(self):
-        self.ha.schedule_reinitialize()
+        self.assertIsNotNone(self.ha.reinitialize())
-        self.ha.schedule_reinitialize()
-        self.ha.run_cycle()
-        self.assertIsNone(self.ha._async_executor.scheduled_action)
 
         self.ha.cluster = get_cluster_initialized_with_leader()
-        self.ha.has_lock = true
+        self.assertIsNone(self.ha.reinitialize())
-        self.ha.schedule_reinitialize()
-        self.ha.run_cycle()
-        self.assertIsNone(self.ha._async_executor.scheduled_action)
 
-        self.ha.has_lock = false
+        self.assertIsNotNone(self.ha.reinitialize())
-        self.ha.schedule_reinitialize()
+
-        self.ha.run_cycle()
+        self.ha.state_handler.name = self.ha.cluster.leader.name
+        self.assertIsNotNone(self.ha.reinitialize())
 
     def test_restart(self):
         self.assertEquals(self.ha.restart(), (True, 'restarted successfully'))
         self.p.restart = false
         self.assertEquals(self.ha.restart(), (False, 'restart failed'))
-        self.ha.schedule_reinitialize()
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.ha.reinitialize()
         self.assertEquals(self.ha.restart(), (False, 'reinitialize already in progress'))
+        with patch.object(self.ha, "restart_matches", return_value=False):
+            self.assertEquals(self.ha.restart({'foo': 'bar'}), (False, "restart conditions are not satisfied"))
 
     def test_restart_in_progress(self):
-        self.ha._async_executor.schedule('restart', True)
+        with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)):
-        self.assertTrue(self.ha.restart_scheduled())
+            self.ha.restart(run_async=True)
-        self.assertEquals(self.ha.run_cycle(), 'not healthy enough for leader race')
+            self.assertTrue(self.ha.restart_scheduled())
+            self.assertEquals(self.ha.run_cycle(), 'not healthy enough for leader race')
 
-        self.ha.cluster = get_cluster_initialized_with_leader()
+            self.ha.cluster = get_cluster_initialized_with_leader()
-        self.assertEquals(self.ha.run_cycle(), 'restart in progress')
+            self.assertEquals(self.ha.run_cycle(), 'restart in progress')
 
-        self.ha.has_lock = true
+            self.ha.has_lock = true
-        self.assertEquals(self.ha.run_cycle(), 'updated leader lock during restart')
+            self.assertEquals(self.ha.run_cycle(), 'updated leader lock during restart')
 
-        self.ha.update_lock = false
+            self.ha.update_lock = false
-        self.assertEquals(self.ha.run_cycle(), 'failed to update leader lock during restart')
+            self.assertEquals(self.ha.run_cycle(), 'failed to update leader lock during restart')
 
     @patch('requests.get', requests_get)
     @patch('time.sleep', Mock())
@@ -335,6 +354,17 @@ class TestHa(unittest.TestCase):
         self.assertEquals('no action.  i am the leader with the lock', self.ha.run_cycle())
 
     @patch('requests.get', requests_get)
+    def test_manual_failover_from_leader_in_pause(self):
+        self.ha.has_lock = true
+        self.ha.is_paused = true
+        scheduled = datetime.datetime.now()
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
+        self.assertEquals('PAUSE: no action.  i am the leader with the lock', self.ha.run_cycle())
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, '', None))
+        self.assertEquals('PAUSE: no action.  i am the leader with the lock', self.ha.run_cycle())
+
+    @patch('requests.get', requests_get)
+    @patch('time.sleep', Mock())
     def test_manual_failover_process_no_leader(self):
         self.p.is_leader = false
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', self.p.name, None))
@@ -359,11 +389,27 @@ class TestHa(unittest.TestCase):
         self.ha.patroni.nofailover = True
         self.assertEquals(self.ha.run_cycle(), 'following a different leader because I am not allowed to promote')
 
+    @patch('time.sleep', Mock())
+    def test_manual_failover_process_no_leader_in_pause(self):
+        self.ha.is_paused = true
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: continue to run as master without lock')
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', '', None))
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: continue to run as master without lock')
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'blabla', None))
+        self.assertEquals('PAUSE: acquired session lock as a leader', self.ha.run_cycle())
+        self.p.is_leader = false
+        self.p.set_role('replica')
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', self.p.name, None))
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: promoted self to leader by acquiring session lock')
+
     def test_is_healthiest_node(self):
         self.ha.state_handler.is_leader = false
         self.ha.patroni.nofailover = False
         self.ha.fetch_node_status = lambda e: (e, True, True, 0, {})
         self.assertTrue(self.ha.is_healthiest_node())
+        self.ha.is_paused = true
+        self.assertFalse(self.ha.is_healthiest_node())
 
     def test__is_healthiest_node(self):
         self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
@@ -395,3 +441,69 @@ class TestHa(unittest.TestCase):
         self.assertEqual(self.ha.post_recover(), 'failed to start postgres')
         self.p.is_running = true
         self.assertIsNone(self.ha.post_recover())
+
+    def test_schedule_future_restart(self):
+        self.ha.patroni.scheduled_restart = {}
+        # do the restart 2 times. The first one should succeed, the second one should fail
+        self.assertTrue(self.ha.schedule_future_restart({'schedule': future_restart_time}))
+        self.assertFalse(self.ha.schedule_future_restart({'schedule': future_restart_time}))
+
+    def test_delete_future_restarts(self):
+        self.ha.delete_future_restart()
+
+    def test_evaluate_scheduled_restart(self):
+        self.p.postmaster_start_time = Mock(return_value=str(postmaster_start_time))
+        # restart while the postmaster has been already restarted, fails
+        with patch.object(self.ha,
+                          'future_restart_scheduled',
+                          Mock(return_value={'postmaster_start_time':
+                                             str(postmaster_start_time - datetime.timedelta(days=1)),
+                                             'schedule': str(future_restart_time)})):
+            self.assertIsNone(self.ha.evaluate_scheduled_restart())
+        with patch.object(self.ha,
+                          'future_restart_scheduled',
+                          Mock(return_value={'postmaster_start_time': str(postmaster_start_time),
+                                             'schedule': str(future_restart_time)})):
+            with patch.object(self.ha,
+                              'should_run_scheduled_action', Mock(return_value=True)):
+                # restart in the future, ok
+                self.assertIsNotNone(self.ha.evaluate_scheduled_restart())
+                with patch.object(self.ha, 'restart', Mock(return_value=(False, "Test"))):
+                    # restart in the future, bit the actual restart failed
+                    self.assertIsNone(self.ha.evaluate_scheduled_restart())
+
+    def test_scheduled_restart(self):
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        with patch.object(self.ha, "evaluate_scheduled_restart", Mock(return_value="restart scheduled")):
+            self.assertEquals(self.ha.run_cycle(), "restart scheduled")
+
+    def test_restart_matches(self):
+        self.p._role = 'replica'
+        self.p.server_version = 90500
+        self.p._pending_restart = True
+        self.assertFalse(self.ha.restart_matches("master", "9.5.0", True))
+        self.assertFalse(self.ha.restart_matches("replica", "9.4.3", True))
+        self.p._pending_restart = False
+        self.assertFalse(self.ha.restart_matches("replica", "9.5.2", True))
+        self.assertTrue(self.ha.restart_matches("replica", "9.5.2", False))
+
+    def test_process_healthy_cluster_in_pause(self):
+        self.p.is_leader = false
+        self.ha.is_paused = true
+        self.p.name = 'leader'
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: removed leader lock because postgres is not running as master')
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None))
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: waiting to become master after promote...')
+
+    def test_postgres_unhealthy_in_pause(self):
+        self.ha.is_paused = true
+        self.p.is_healthy = false
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: postgres is not running')
+        self.ha.has_lock = true
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: removed leader lock because postgres is not running')
+
+    def test_no_etcd_connection_in_pause(self):
+        self.ha.is_paused = true
+        self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: DCS is not accessible')

tests/test_patroni.py

@@ -6,6 +6,7 @@ import unittest
 from mock import Mock, patch
 from patroni.api import RestApiServer
 from patroni.async_executor import AsyncExecutor
+from patroni.dcs.etcd import Client
 from patroni.exceptions import DCSError
 from patroni import Patroni, main as _main
 from six.moves import BaseHTTPServer
@@ -13,6 +14,11 @@ from test_etcd import SleepException, etcd_read, etcd_write
 from test_postgresql import Postgresql, psycopg2_connect
 
 
+class MockFrozenImporter(object):
+
+    toc = set(['patroni.dcs.etcd'])
+
+
 @patch('time.sleep', Mock())
 @patch('subprocess.call', Mock(return_value=0))
 @patch('psycopg2.connect', psycopg2_connect)
@@ -26,12 +32,14 @@ from test_postgresql import Postgresql, psycopg2_connect
 @patch.object(etcd.Client, 'read', etcd_read)
 class TestPatroni(unittest.TestCase):
 
+    @patch('pkgutil.get_importer', Mock(return_value=MockFrozenImporter()))
+    @patch('sys.frozen', Mock(return_value=True), create=True)
     @patch.object(etcd.Client, 'read', etcd_read)
     def setUp(self):
         RestApiServer._BaseServer__is_shut_down = Mock()
         RestApiServer._BaseServer__shutdown_request = True
         RestApiServer.socket = 0
-        with patch.object(etcd.Client, 'machines') as mock_machines:
+        with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
             sys.argv = ['patroni.py', 'postgres0.yml']
             self.p = Patroni()
@@ -44,7 +52,7 @@ class TestPatroni(unittest.TestCase):
 
     @patch('time.sleep', Mock(side_effect=SleepException))
     @patch.object(etcd.Client, 'delete', Mock())
-    @patch.object(etcd.Client, 'machines')
+    @patch.object(Client, 'machines')
     def test_patroni_main(self, mock_machines):
         with patch('subprocess.call', Mock(return_value=1)):
             sys.argv = ['patroni.py', 'postgres0.yml']
@@ -53,7 +61,8 @@ class TestPatroni(unittest.TestCase):
             with patch.object(Patroni, 'run', Mock(side_effect=SleepException)):
                 self.assertRaises(SleepException, _main)
             with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())):
-                _main()
+                with patch('patroni.ha.Ha.is_paused', Mock(return_value=True)):
+                    _main()
 
     @patch('patroni.config.Config.save_cache', Mock())
     @patch('patroni.config.Config.reload_local_configuration', Mock(return_value=True))
@@ -74,7 +83,7 @@ class TestPatroni(unittest.TestCase):
     def test_schedule_next_run(self):
         self.p.ha.dcs.watch = Mock(return_value=True)
         self.p.schedule_next_run()
-        self.p.next_run = time.time() - self.p.nap_time - 1
+        self.p.next_run = time.time() - self.p.dcs.loop_wait - 1
         self.p.schedule_next_run()
 
     def test_noloadbalance(self):

tests/test_postgresql.py

@@ -32,8 +32,10 @@ class MockCursor(object):
             self.results = [(0,)]
         elif sql == 'SELECT pg_is_in_recovery()':
             self.results = [(False, )]
-        elif sql.startswith('SELECT to_char(pg_postmaster_start_time'):
+        elif sql.startswith('WITH replication_info AS ('):
-            self.results = [('', True, '', '', '', '', False)]
+            replication_info = '[{"application_name":"walreceiver","client_addr":"1.2.3.4",' +\
+                               '"state":"streaming","sync_state":"async","sync_priority":0}]'
+            self.results = [('', True, '', '', '', '', False, replication_info)]
         elif sql.startswith('SELECT name, setting'):
             self.results = [('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
                             ('search_path', 'public', None, 'string', 'user'),
@@ -182,7 +184,7 @@ class TestPostgresql(unittest.TestCase):
                              'restore': 'true'})
         self.leadermem = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres'})
         self.leader = Leader(-1, 28, self.leadermem)
-        self.other = Member(0, 'test1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres',
+        self.other = Member(0, 'test-1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres',
                             'tags': {'replicatefrom': 'leader'}})
         self.me = Member(0, 'test0', 28, {'conn_url': 'postgres://replicator:[email protected]:5434/postgres'})
 
@@ -263,22 +265,18 @@ class TestPostgresql(unittest.TestCase):
         with patch.object(Postgresql, 'restart', Mock(return_value=False)):
             self.p.set_role('replica')
             self.p.follow(None, None)  # restart without rewind
-            self.p.set_role('master')
 
         with patch.object(Postgresql, 'stop', Mock(return_value=False)):
-            self.p.follow(self.leader, self.leader)  # failed to stop postgres
+            self.p.follow(self.leader, self.leader, need_rewind=True)  # failed to stop postgres
-
-        self.p.follow(self.leader, None)  # Leader unknown, can not rewind
 
         self.p.follow(self.leader, self.leader)  # "leader" is not accessible or is_in_recovery
 
         with patch.object(Postgresql, 'checkpoint', Mock(return_value=None)):
             self.p.follow(self.leader, self.leader)
-            self.p.set_role('master')
             mock_pg_rewind.return_value = True
-            self.p.follow(self.leader, self.leader)
+            self.p.follow(self.leader, self.leader, need_rewind=True)
 
-        self.assertTrue(self.p.follow(None, None))  # check_recovery_conf...
+        self.p.follow(None, None)  # check_recovery_conf...
 
     @patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string))
     def test_can_rewind(self):
@@ -313,13 +311,20 @@ class TestPostgresql(unittest.TestCase):
     def test_sync_replication_slots(self):
         self.p.start()
         cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None)
+        with mock.patch('patroni.postgresql.Postgresql._query', Mock(side_effect=psycopg2.OperationalError)):
+            self.p.sync_replication_slots(cluster)
         self.p.sync_replication_slots(cluster)
-        self.p.query = Mock(side_effect=psycopg2.OperationalError)
-        self.p.schedule_load_slots = True
-        self.p.sync_replication_slots(cluster)
-        self.p.schedule_load_slots = False
         with mock.patch('patroni.postgresql.Postgresql.role', new_callable=PropertyMock(return_value='replica')):
             self.p.sync_replication_slots(cluster)
+        with mock.patch('patroni.postgresql.logger.error', new_callable=Mock()) as errorlog_mock:
+            self.p.query = Mock()
+            alias1 = Member(0, 'test-3', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres'})
+            alias2 = Member(0, 'test.3', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres'})
+            cluster.members.extend([alias1, alias2])
+            self.p.sync_replication_slots(cluster)
+            errorlog_mock.assert_called_once()
+            assert "test-3" in errorlog_mock.call_args[0][1]
+            assert "test.3" in errorlog_mock.call_args[0][1]
 
     @patch.object(MockConnect, 'closed', 2)
     def test__query(self):
@@ -537,3 +542,9 @@ class TestPostgresql(unittest.TestCase):
             self.assertEquals(self.p.get_major_version(), 9.4)
         with patch.object(builtins, 'open', Mock(side_effect=Exception)):
             self.assertEquals(self.p.get_major_version(), 0.0)
+
+    def test_postmaster_start_time(self):
+        with patch.object(MockCursor, "fetchone", Mock(return_value=('foo', True, '', '', '', '', False))):
+            self.assertEqual(self.p.postmaster_start_time(), 'foo')
+        with patch.object(MockCursor, "execute", side_effect=psycopg2.Error):
+            self.assertIsNone(self.p.postmaster_start_time())

tests/test_wale_restore.py

@@ -6,30 +6,10 @@ from mock import MagicMock, patch, PropertyMock
 from patroni.scripts.wale_restore import WALERestore, main as _main
 
 
-def fake_backup_data(self, *args, **kwargs):
+wale_output = b'name last_modified expanded_size_bytes wal_segment_backup_start ' +\
-    """ return the fake result of WAL-E backup-list"""
+              b'wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop\n' +\
-    return """name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop
+              b'base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 ' +\
-base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   00000001000000000000007F    00000040 00000001000000000000007F    00000240
+              b'00000001000000000000007F 00000040 00000001000000000000007F 00000240\n'
-"""
-
-
-def fake_backup_data_2(self, *args, **kwargs):
-    """ return the fake result of WAL-E backup-list"""
-    return """name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop """
-
-
-def fake_backup_data_3(self, *args, **kwargs):
-    """ return the fake result of WAL-E backup-list"""
-    return """name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop
-base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   00000001000000000000007F    00000040 00000001000000000000007F    00000240
-"""
-
-
-def fake_backup_data_4(self, *args, **kwargs):
-    """ return the fake result of WAL-E backup-list"""
-    return """name    last_modified   expanded_size_foo wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop
-base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   00000001000000000000007F    00000040 00000001000000000000007F    00000240
-"""
 
 
 @patch('os.access', MagicMock(return_value=True))
@@ -39,7 +19,7 @@ base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   000
 @patch('psycopg2.extensions.cursor', MagicMock(autospec=True))
 @patch('psycopg2.extensions.connection', MagicMock(autospec=True))
 @patch('psycopg2.connect', MagicMock(autospec=True))
-@patch('subprocess.check_output', MagicMock(side_effect=fake_backup_data))
+@patch('subprocess.check_output', MagicMock(return_value=wale_output))
 class TestWALERestore(unittest.TestCase):
 
     def setUp(self):
@@ -50,11 +30,13 @@ class TestWALERestore(unittest.TestCase):
             self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
         with patch('subprocess.check_output', MagicMock(side_effect=subprocess.CalledProcessError(1, "cmd", "foo"))):
             self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output', MagicMock(side_effect=fake_backup_data_2)):
+        with patch('subprocess.check_output', MagicMock(return_value=wale_output.split(b'\n')[0])):
             self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output', MagicMock(side_effect=fake_backup_data_3)):
+        with patch('subprocess.check_output',
+                   MagicMock(return_value=wale_output.replace(b' wal_segment_offset_backup_stop', b''))):
             self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output', MagicMock(side_effect=fake_backup_data_4)):
+        with patch('subprocess.check_output',
+                   MagicMock(return_value=wale_output.replace(b'expanded_size_bytes', b'expanded_size_foo'))):
             self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
 
         self.wale_restore.should_use_s3_to_create_replica()

tests/test_zookeeper.py

@@ -58,11 +58,16 @@ class MockKazooClient(Mock):
             raise TypeError("Invalid type for 'path' (string expected)")
         if not isinstance(value, (six.binary_type,)):
             raise TypeError("Invalid type for 'value' (must be a byte string)")
+        if value == b'Exception':
+            raise Exception
         if path.endswith('/initialize') or path == '/service/test/optime/leader':
             raise Exception
         elif value == b'retry' or (value == b'exists' and self.exists):
             raise NodeExistsError
 
+    def create_async(self, path, value=b"", acl=None, ephemeral=False, sequence=False, makepath=False):
+        return self.create(path, value, acl, ephemeral, sequence, makepath) or Mock()
+
     @staticmethod
     def set(path, value, version=-1):
         if not isinstance(path, six.string_types):
@@ -80,6 +85,9 @@ class MockKazooClient(Mock):
                 return
         raise NoNodeError
 
+    def set_async(self, path, value, version=-1):
+        return self.set(path, value, version) or Mock()
+
     def delete(self, path, version=-1, recursive=False):
         if not isinstance(path, six.string_types):
             raise TypeError("Invalid type for 'path' (string expected)")
@@ -92,6 +100,9 @@ class MockKazooClient(Mock):
         elif path.endswith('/') or path.endswith('/initialize') or path == '/service/test/members/bar':
             raise NoNodeError
 
+    def delete_async(self, path, version=-1, recursive=False):
+        return self.delete(path, version, recursive) or Mock()
+
 
 class TestPatroniSequentialThreadingHandler(unittest.TestCase):
 
@@ -109,16 +120,14 @@ class TestZooKeeper(unittest.TestCase):
     @patch('patroni.dcs.zookeeper.KazooClient', MockKazooClient)
     def setUp(self):
         self.zk = ZooKeeper({'hosts': ['localhost:2181'], 'scope': 'test',
-                             'name': 'foo', 'ttl': 30, 'retry_timeout': 10})
+                             'name': 'foo', 'ttl': 30, 'retry_timeout': 10, 'loop_wait': 10})
 
     def test_session_listener(self):
         self.zk.session_listener(KazooState.SUSPENDED)
 
-    def test_set_ttl(self):
+    def test_reload_config(self):
-        self.zk.set_ttl(20)
+        self.zk.reload_config({'ttl': 20, 'retry_timeout': 10, 'loop_wait': 10})
-
+        self.zk.reload_config({'ttl': 20, 'retry_timeout': 10, 'loop_wait': 5})
-    def test_set_retry_timeout(self):
-        self.zk.set_retry_timeout(10)
 
     def test_get_node(self):
         self.assertIsNone(self.zk.get_node('/no_node'))
@@ -165,7 +174,7 @@ class TestZooKeeper(unittest.TestCase):
         self.zk.touch_member('new')
         self.zk._name = 'na'
         self.zk._client.exists = 1
-        self.zk.touch_member('exists')
+        self.zk.touch_member('Exception')
         self.zk._name = 'bar'
         self.zk.touch_member('retry')
         self.zk._fetch_cluster = True
@@ -183,8 +192,12 @@ class TestZooKeeper(unittest.TestCase):
     def test_write_leader_optime(self):
         self.zk.last_leader_operation = '0'
         self.zk.write_leader_optime('1')
+        with patch.object(MockKazooClient, 'create_async', Mock()):
+            self.zk.write_leader_optime('1')
+        with patch.object(MockKazooClient, 'set_async', Mock()):
+            self.zk.write_leader_optime('2')
         self.zk._base_path = self.zk._base_path.replace('test', 'bla')
-        self.zk.write_leader_optime('2')
+        self.zk.write_leader_optime('3')
 
     def test_delete_cluster(self):
         self.assertTrue(self.zk.delete_cluster())
@@ -193,3 +206,8 @@ class TestZooKeeper(unittest.TestCase):
         self.zk.watch(0)
         self.zk.event.isSet = lambda: True
         self.zk.watch(0)
+
+    def test__kazoo_connect(self):
+        self.zk._client._retry.deadline = 1
+        self.zk._orig_kazoo_connect = Mock(return_value=(0, 0))
+        self.zk._kazoo_connect(None, None)