Fix rewind behavior when paused (#365)

* Check if we can rewind when deciding to call Postgresql.follow during pause.

Without this the following sequence of events occurs:

1. Manual failover from node, demotion sets need_rewind flag. Rewind is not done because can_rewind is False.
2. Cluster is put into paused mode. Rewind is not attempted because recovery.conf matches.
3. PostgreSQL goes down (pg_ctl stop).
4. Because need_rewind is set PostgreSQL is restarted.
5. Sysadmin is unhappy because Patroni is doing stuff behind his back during pause.

* Allow superuser user to be missing from config for rewind.

In other places a missing superuser authentication section is allowed and we default to libpq's use default OS user with no password. This makes rewind work with a missing superuser configuration.

patroni/ha.py

@@ -208,7 +208,7 @@ class Ha(object):
 
         node_to_follow = self._get_node_to_follow(self.cluster)
 
-        if self.is_paused() and not self.state_handler.need_rewind:
+        if self.is_paused() and not (self.state_handler.need_rewind and self.state_handler.can_rewind):
             self.state_handler.set_role('master' if is_leader else 'replica')
             if is_leader:
                 return 'continue to run as master without lock'
@@ -900,7 +900,7 @@ class Ha(object):
                         self.dcs.delete_leader()
                         self.dcs.reset_cluster()
                         return 'removed leader lock because postgres is not running'
-                    elif not self.state_handler.need_rewind:
+                    elif not (self.state_handler.need_rewind and self.state_handler.can_rewind):
                         return 'postgres is not running'
 
                 # try to start dead postgres

patroni/postgresql.py

@@ -469,6 +469,9 @@ class Postgresql(object):
             os.unlink(self._trigger_file)
 
     def write_pgpass(self, record):
+        if 'user' not in record or 'password' not in record:
+            return os.environ.copy()
+
         with open(self._pgpass, 'w') as f:
             os.fchmod(f.fileno(), 0o600)
             f.write('{host}:{port}:*:{user}:{password}\n'.format(**record))
@@ -905,7 +908,15 @@ class Postgresql(object):
     def rewind(self, r):
         # prepare pg_rewind connection
         env = self.write_pgpass(r)
-        dsn = 'user={user} host={host} port={port} dbname={database} sslmode=prefer sslcompression=1'.format(**r)
+        dsn_attrs = [
+            ('user', r.get('user')),
+            ('host', r.get('host')),
+            ('port', r.get('port')),
+            ('dbname', r.get('database')),
+            ('sslmode', 'prefer'),
+            ('sslcompression', '1'),
+        ]
+        dsn = " ".join("{0}={1}".format(k, v) for k, v in dsn_attrs if v is not None)
         logger.info('running pg_rewind from %s', dsn)
         try:
             return subprocess.call([self._pgcommand('pg_rewind'),