More standby cluster bugfixes (#1053)

1. use the default port is 5432 when only standby_cluster.host is defined
2. check that standby_cluster replica can be bootstrapped without connection to the standby_cluster leader against `create_replica_methods` defined in the `standby_cluster` config instead of the `postgresql` section.
3. Don't fallback to the create_replica_methods defined in the `postgresql` section when bootstrapping a member of the standby cluster.
4. Make sure we specify the database when connecting to the leader.

patroni/dcs/__init__.py

@@ -136,7 +136,7 @@ class Member(namedtuple('Member', 'index,name,session,data')):
         if conn_kwargs:
             conn_url = 'postgresql://{host}:{port}'.format(
                 host=conn_kwargs.get('host'),
-                port=conn_kwargs.get('port'),
+                port=conn_kwargs.get('port', 5432),
             )
             self.data['conn_url'] = conn_url
             return conn_url

patroni/ha.py

@@ -251,12 +251,14 @@ class Ha(object):
             else:
                 return 'failed to acquire initialize lock'
         else:
-            if self.state_handler.can_create_replica_without_replication_connection():
+            create_replica_methods = self.get_standby_cluster_config().get('create_replica_methods', []) \
+                                     if self.is_standby_cluster() else None
+            if self.state_handler.can_create_replica_without_replication_connection(create_replica_methods):
                 msg = 'bootstrap (without leader)'
                 self._async_executor.schedule(msg)
                 self._async_executor.run_async(self.clone)
                 return 'trying to ' + msg
-            return 'waiting for leader to bootstrap'
+            return 'waiting for {0}leader to bootstrap'.format('standby_' if self.is_standby_cluster() else '')
 
     def bootstrap_standby_leader(self):
         """ If we found 'standby' key in the configuration, we need to bootstrap

patroni/postgresql.py

@@ -666,11 +666,12 @@ class Postgresql(object):
     def replica_method_can_work_without_replication_connection(self, method):
         return method != 'basebackup' and self.config and self.config.get(method, {}).get('no_master')
 
-    def can_create_replica_without_replication_connection(self):
+    def can_create_replica_without_replication_connection(self, replica_methods=None):
         """ go through the replication methods to see if there are ones
             that does not require a working replication connection.
         """
-        replica_methods = self._create_replica_methods
+        if replica_methods is None:
+            replica_methods = self._create_replica_methods
         return any(self.replica_method_can_work_without_replication_connection(method) for method in replica_methods)
 
     def create_replica(self, clone_member):
@@ -684,16 +685,12 @@ class Postgresql(object):
         self._sysid = None
 
         is_remote_master = isinstance(clone_member, RemoteMember)
-        create_replica_methods = is_remote_master and clone_member.create_replica_methods
 
         # get list of replica methods either from clone member or from
         # the config. If there is no configuration key, or no value is
         # specified, use basebackup
-        replica_methods = (
-            create_replica_methods
-            or self._create_replica_methods
-            or ['basebackup']
-        )
+        replica_methods = (clone_member.create_replica_methods if is_remote_master
+                           else self._create_replica_methods) or ['basebackup']
 
         if clone_member and clone_member.conn_url:
             r = clone_member.conn_kwargs(self._replication)
@@ -1310,6 +1307,8 @@ class Postgresql(object):
             yield cur
 
     def check_leader_is_not_in_recovery(self, **kwargs):
+        if not kwargs.get('database'):
+            kwargs['database'] = self._database
         try:
             with self._get_connection_cursor(connect_timeout=3, options='-c statement_timeout=2000', **kwargs) as cur:
                 cur.execute('SELECT pg_catalog.pg_is_in_recovery()')

tests/test_ha.py

@@ -206,13 +206,19 @@ class TestHa(unittest.TestCase):
         self.assertEqual(self.ha.run_cycle(), 'starting as a secondary')
 
     @patch('patroni.dcs.etcd.Etcd.initialize', return_value=True)
-    def test_start_as_standby_leader(self, initialize):
+    def test_bootstrap_as_standby_leader(self, initialize):
         self.p.data_directory_empty = true
         self.ha.cluster = get_cluster_not_initialized_without_leader(cluster_config=ClusterConfig(0, {}, 0))
         self.ha.cluster.is_unlocked = true
         self.ha.patroni.config._dynamic_configuration = {"standby_cluster": {"port": 5432}}
         self.assertEqual(self.ha.run_cycle(), 'trying to bootstrap a new standby leader')
 
+    def test_bootstrap_waiting_for_standby_leader(self):
+        self.p.data_directory_empty = true
+        self.ha.cluster = get_cluster_initialized_without_leader()
+        self.ha.cluster.config.data.update({'standby_cluster': {'port': 5432}})
+        self.assertEqual(self.ha.run_cycle(), 'waiting for standby_leader to bootstrap')
+
     @patch.object(Cluster, 'get_clone_member',
                   Mock(return_value=Member(0, 'test', 1, {'api_url': 'http://127.0.0.1:8011/patroni',
                                                           'conn_url': 'postgres://127.0.0.1:5432/postgres'})))