Add HA loop diagram (#352)

docs/ha_loop_diagram.dot

@@ -0,0 +1,116 @@
+// Graphviz source for ha_loop_diagram.png
+// recompile with:
+// dot -Tpng ha_loop_diagram.dot -o ha_loop_diagram.png
+
+digraph G {
+  rankdir=TB;
+  fontname="sans-serif";
+  penwidth="0.3";
+  layout="dot";
+  newrank=true;
+  edge [fontname="sans-serif", 
+        fontsize=12, 
+        color=black, 
+        fontcolor=black];
+  node [fontname=serif, 
+        fontsize=12, 
+        fillcolor=white, 
+        color=black, 
+        fontcolor=black,
+        style=filled];
+  "start" [label=Start, shape="rectangle", fillcolor="green"]
+  "start" -> "load_cluster_from_dcs";
+  
+  subgraph cluster_run_cycle {
+    label="run_cycle"
+    "load_cluster_from_dcs" [label="Load cluster from DCS"];
+    "touch_member" [label="Persist node in DCS"];
+    "cluster.has_member" [shape="diamond", label="Is node registered on DCS?"]
+    "cluster.has_member" -> "touch_member"  [label="no" color="red"];
+    "is_initialized" [shape="oval", 
+                      label="Ensure cluster is initialized on DCS"];
+    "has_lock" [label="Did I win the race to the leader lock?", shape=diamond];
+    "has_lock" -> "is_initialized"  [label="yes" color="green"];
+    "has_lock" -> "handle_long_action_in_progress"  [label="no" color="red"];
+    "load_cluster_from_dcs" -> "cluster.has_member";
+    "touch_member" -> "has_lock";
+    "cluster.has_member" -> "has_lock"  [label="yes" color="green"];
+    "handle_long_action_in_progress" [label="Wait for async operations to finish"];
+    "is_initialized" -> "handle_long_action_in_progress";
+    "handle_long_action_in_progress" -> "recovering?";
+    "recovering?" [label="Was cluster recovering and failed?", shape="diamond"];
+    "recovering?" -> "post_recover" [label="yes" color="green"];
+    "recovering?" -> "data_directory_empty" [label="no" color="red"];
+    "post_recover" [label="Remove leader key"];
+    "data_directory_empty" [label="Is data folder empty?", shape="diamond"];
+    "data_directory_empty" -> "bootstrap" [label="yes" color="green"];
+    "bootstrap" [label="Try to bootstrap node from leader\n(async)"]
+    "data_directory_empty" -> "data_belongs_to_cluster" [label="no" color="red"];
+    "data_belongs_to_cluster" [label="Does data dir belong to cluster?", shape="diamond"];
+    "data_belongs_to_cluster" -> "cluster_initialize" [label="yes" color="green"];
+    "data_belongs_to_cluster" -> "exit" [label="no" color="red"];
+    "cluster_initialize" [label="Is cluster initialized on DCS?" shape="diamond"]
+    "cluster_initialize" -> "dcs.initialize" [label="no" color="red"]
+    "cluster_initialize" -> "is_healthy" [label="yes" color="green"]
+    "dcs.initialize" [label="Initialize new cluster"];
+    "dcs.initialize" -> "is_healthy"
+    "is_healthy" [label="Is node healthy?\n(running Postgres)", shape="diamond"];
+    "recover" [label="Restart as read-only\nand set Recover flag"]
+    "is_healthy" -> "recover" [label="no" color="red"]; 
+    "is_healthy" -> "cluster.is_unlocked" [label="yes" color="green"]; 
+    "cluster.is_unlocked" [label="Does the cluster have a leader?", shape="diamond"]
+  }
+
+  "cluster.is_unlocked" -> "unhealthy_is_healthiest" [label="no" color="red"]
+  "cluster.is_unlocked" -> "healthy_has_lock" [label="yes" color="green"]
+  "bootstrap" -> "start"
+  "recover" -> "start"
+  "post_recover" -> "start";
+
+  subgraph cluster_process_healthy_cluster {
+    label = "process_healthy_cluster"
+    "healthy_has_lock" [label="Did I win the race to the leader lock?", shape=diamond]
+    "healthy_is_leader" [label="Is Postgres running as master?", shape=diamond]
+    "healthy_no_lock" [label="Is Postgres running as master?", shape=diamond]
+    "healthy_no_lock" -> "healthy_demote" [label="yes" color="green"]
+    "healthy_has_lock" -> "healthy_no_lock" [label="no" color="red"]
+    "healthy_has_lock" -> "healthy_is_leader" [label="yes" color="green"]
+    "healthy_failover" [label="Promote Postgres to master"]
+    "healthy_is_leader" -> "healthy_failover" [label="no" color="red"]
+    "healthy_is_leader" -> "renew_lock" [label="yes" color="green"]
+    "healthy_failover" -> "renew_lock"
+    "renew_lock" [label="Renew leader lock"]
+    "healthy_demote" [label="Demote Postgres to secondary"]
+    "healthy_no_lock" -> "healthy_start" [label="no" color="red"]
+    "healthy_demote" -> "healthy_start"
+    "renew_lock" -> "healthy_start"
+    "healthy_start" [label="Go to start", shape=rectangle, fillcolor=green]
+  }
+
+  subgraph cluster_process_unhealthy_cluster {
+    label = "process_unhealthy_cluster"
+    "unhealthy_is_healthiest" [label="Am I the healthiest node?", shape="diamond"]
+    "unhealthy_is_healthiest" -> "unhealthy_acquire_lock" [label="yes, try to get lock"]
+    "unhealthy_acquire_lock" [label="Was I able to get the lock?", shape="diamond"]
+    "unhealthy_acquire_lock" -> "unhealthy_failover_key" [label="yes" color="green"]
+    "unhealthy_failover_key" [label="Is there a failover key set?", shape="diamond"]
+    "unhealthy_failover_key" -> "unhealthy_clean_failover_key" [label="yes" color="green"]
+    "unhealthy_failover_key" -> "unhealthy_promote" [label="yes" color="green"]
+    "unhealthy_clean_failover_key" [label="Clean failover key"]
+    "unhealthy_clean_failover_key" -> unhealthy_promote
+    "unhealthy_promote" [label="Promote to master"]
+    "unhealthy_is_healthiest" -> "unhealthy_need_rewind" [label="no" color="red"]
+    "unhealthy_need_rewind" [label="Do I need to be rewinded?", shape="diamond"]
+    "unhealthy_need_rewind" -> "unhealthy_set_rewind_flag" [label="yes" color="green"]
+    "unhealthy_need_rewind" -> "unhealthy_demote" [label="no" color="red"]
+    "unhealthy_set_rewind_flag" [label="Set rewind flag"]
+    unhealthy_set_rewind_flag -> unhealthy_demote
+    "unhealthy_demote" [label="Demote and follow somebody else"]
+    "unhealthy_demote" -> "unhealthy_start"
+    "unhealthy_acquire_lock" -> "unhealthy_start" [label="no" color="red"]
+    "unhealthy_promote" -> "unhealthy_start"
+    "unhealthy_start" [label="Go to start", shape=rectangle, fillcolor=green]
+  }
+
+  "exit" [label="Fail and exit", fillcolor=red]
+}