Fix callbacks behavior (mostly for standby cluster) (#998)

First of all, this patch changes the behavior of `on_start`/`on_restart` callbacks, they will be called only when postgres is started or restarted without role changes. In case if the member is promoted or demoted only the `on_role_change` callback will be executed. `on_role_change` was never called for standby leader, only `on_start`/`on_restart` and with a wrong role argument.
Before that `on_role_change` was never called for standby leader, only `on_start`/`on_restart` and with a wrong role argument.

In addition to that, the REST API will return standby_leader role for the leader of the standby cluster.

Closes https://github.com/zalando/patroni/issues/988

features/environment.py

@@ -111,9 +111,9 @@ class PatroniController(AbstractController):
         with open(os.path.join(self._data_dir, 'label'), 'w') as f:
             f.write(content)
 
-    def read_label(self):
+    def read_label(self, label):
         try:
-            with open(os.path.join(self._data_dir, 'label'), 'r') as f:
+            with open(os.path.join(self._data_dir, label), 'r') as f:
                 return f.read().strip()
         except IOError:
             return None

features/standby_cluster.feature

@@ -2,11 +2,11 @@ Feature: standby cluster
   Scenario: check permanent logical slots are preserved on failover/switchover
     Given I start postgres1
     Then postgres1 is a leader after 10 seconds
-    And I sleep for 2 seconds
+    And I sleep for 3 seconds
     When I issue a PATCH request to http://127.0.0.1:8009/config with {"loop_wait": 2, "slots": {"pm_1": {"type": "physical"}}, "postgresql": {"parameters": {"wal_level": "logical"}}}
     Then I receive a response code 200
     And Response on GET http://127.0.0.1:8009/config contains slots after 10 seconds
-    And I sleep for 2 seconds
+    And I sleep for 3 seconds
     When I issue a PATCH request to http://127.0.0.1:8009/config with {"slots": {"test_logical": {"type": "logical", "database": "postgres", "plugin": "test_decoding"}}}
     Then I receive a response code 200
     When I start postgres0 with callback configured
@@ -14,7 +14,7 @@ Feature: standby cluster
     And replication works from postgres1 to postgres0 after 15 seconds
     When I shut down postgres1
     Then postgres0 is a leader after 10 seconds
-    And I sleep for 2 seconds
+    And "members/postgres0" key in DCS has role=master after 3 seconds
     When I issue a GET request to http://127.0.0.1:8008/
     Then I receive a response code 200
     And there is a label with "test_logical" in postgres0 data directory
@@ -24,6 +24,12 @@ Feature: standby cluster
     Then postgres1 is a leader of batman1 after 10 seconds
     When I add the table foo to postgres0
     Then table foo is present on postgres1 after 20 seconds
+    When I issue a GET request to http://127.0.0.1:8009/master
+    Then I receive a response code 200
+    When I issue a GET request to http://127.0.0.1:8009/standby_leader
+    Then I receive a response code 200
+    And I receive a response role standby_leader
+    And there is a postgres1_cb.log with "on_start replica batman1\non_role_change standby_leader batman1" in postgres1 data directory
     When I start postgres2 in a cluster batman1
     Then postgres2 role is the replica after 24 seconds
     And table foo is present on postgres2 after 20 seconds
@@ -31,4 +37,11 @@ Feature: standby cluster
   Scenario: check failover
     When I kill postgres1
     And I kill postmaster on postgres1
-    Then postgres2 is replicating from postgres0 after 20 seconds
+    Then postgres2 is replicating from postgres0 after 32 seconds
+    When I issue a GET request to http://127.0.0.1:8010/master
+    Then I receive a response code 200
+    When I issue a GET request to http://127.0.0.1:8010/standby_leader
+    Then I receive a response code 200
+    And I receive a response role standby_leader
+    And replication works from postgres0 to postgres2 after 15 seconds
+    And there is a postgres2_cb.log with "on_start replica batman1\non_role_change standby_leader batman1" in postgres2 data directory

features/steps/cascading_replication.py

@@ -9,9 +9,10 @@ def start_patroni_with_a_name_value_tag(context, name, tag_name, tag_value):
     return context.pctl.start(name, custom_config={'tags': {tag_name: tag_value}})
 
 
-@then('There is a label with "{content:w}" in {name:w} data directory')
-def check_label(context, content, name):
-    label = context.pctl.read_label(name)
+@then('There is a {label} with "{content}" in {name:w} data directory')
+def check_label(context, label, content, name):
+    label = context.pctl.read_label(name, label)
+    label = label.replace('\n', '\\n')
     assert label == content, "{0} is not equal to {1}".format(label, content)
 
 

features/steps/standby_cluster.py

@@ -9,6 +9,8 @@ SELECT * FROM pg_catalog.pg_stat_replication
 WHERE application_name = '{0}'
 """
 
+callback = "bash -c 'echo \"${*: -3:1} ${*: -2:1} ${*: -1:1}\" >> data/$1/$1_cb.log' -- "
+
 
 @step('I start {name:w} with callback configured')
 def start_patroni_with_callbacks(context, name):
@@ -24,12 +26,15 @@ def start_patroni_with_callbacks(context, name):
 @step('I start {name:w} in a cluster {cluster_name:w}')
 def start_patroni(context, name, cluster_name):
     return context.pctl.start(name, custom_config={
-        "scope": cluster_name
+        "scope": cluster_name,
+        "postgresql": {
+            "callbacks": {c: callback + name for c in ('on_start', 'on_stop', 'on_restart', 'on_role_change')}
+        }
     })
 
 
 @step('I start {name:w} in a standby cluster {cluster_name:w} as a clone of {name2:w}')
-def start_patroni_stanby_cluster(context, name, cluster_name, name2):
+def start_patroni_standby_cluster(context, name, cluster_name, name2):
     # we need to remove patroni.dynamic.json in order to "bootstrap" standby cluster with existing PGDATA
     os.unlink(os.path.join(context.pctl._processes[name]._data_dir, 'patroni.dynamic.json'))
     port = context.pctl._processes[name2]._connkwargs.get('port')
@@ -37,12 +42,18 @@ def start_patroni_stanby_cluster(context, name, cluster_name, name2):
         "scope": cluster_name,
         "bootstrap": {
             "dcs": {
+                "ttl": 20,
+                "loop_wait": 2,
+                "retry_timeout": 5,
                 "standby_cluster": {
                     "host": "localhost",
                     "port": port,
                     "primary_slot_name": "pm_1",
                 }
             }
+        },
+        "postgresql": {
+            "callbacks": {c: callback + name for c in ('on_start', 'on_stop', 'on_restart', 'on_role_change')}
         }
     })
     return context.pctl.start(name)
@@ -60,8 +71,8 @@ def check_replication_status(context, pg_name1, pg_name2, timeout):
         )
 
         if cur and len(cur.fetchall()) != 0:
-            return True
+            break
 
         time.sleep(1)
-
-    return False
+    else:
+        assert False, "{0} is not replicating from {1} after {2} seconds".format(pg_name1, pg_name2, timeout)