diff --git a/docs/ha_loop_diagram.dot b/docs/ha_loop_diagram.dot new file mode 100644 index 00000000..24c8d409 --- /dev/null +++ b/docs/ha_loop_diagram.dot @@ -0,0 +1,116 @@ +// Graphviz source for ha_loop_diagram.png +// recompile with: +// dot -Tpng ha_loop_diagram.dot -o ha_loop_diagram.png + +digraph G { + rankdir=TB; + fontname="sans-serif"; + penwidth="0.3"; + layout="dot"; + newrank=true; + edge [fontname="sans-serif", + fontsize=12, + color=black, + fontcolor=black]; + node [fontname=serif, + fontsize=12, + fillcolor=white, + color=black, + fontcolor=black, + style=filled]; + "start" [label=Start, shape="rectangle", fillcolor="green"] + "start" -> "load_cluster_from_dcs"; + + subgraph cluster_run_cycle { + label="run_cycle" + "load_cluster_from_dcs" [label="Load cluster from DCS"]; + "touch_member" [label="Persist node in DCS"]; + "cluster.has_member" [shape="diamond", label="Is node registered on DCS?"] + "cluster.has_member" -> "touch_member" [label="no" color="red"]; + "is_initialized" [shape="oval", + label="Ensure cluster is initialized on DCS"]; + "has_lock" [label="Did I win the race to the leader lock?", shape=diamond]; + "has_lock" -> "is_initialized" [label="yes" color="green"]; + "has_lock" -> "handle_long_action_in_progress" [label="no" color="red"]; + "load_cluster_from_dcs" -> "cluster.has_member"; + "touch_member" -> "has_lock"; + "cluster.has_member" -> "has_lock" [label="yes" color="green"]; + "handle_long_action_in_progress" [label="Wait for async operations to finish"]; + "is_initialized" -> "handle_long_action_in_progress"; + "handle_long_action_in_progress" -> "recovering?"; + "recovering?" [label="Was cluster recovering and failed?", shape="diamond"]; + "recovering?" -> "post_recover" [label="yes" color="green"]; + "recovering?" -> "data_directory_empty" [label="no" color="red"]; + "post_recover" [label="Remove leader key"]; + "data_directory_empty" [label="Is data folder empty?", shape="diamond"]; + "data_directory_empty" -> "bootstrap" [label="yes" color="green"]; + "bootstrap" [label="Try to bootstrap node from leader\n(async)"] + "data_directory_empty" -> "data_belongs_to_cluster" [label="no" color="red"]; + "data_belongs_to_cluster" [label="Does data dir belong to cluster?", shape="diamond"]; + "data_belongs_to_cluster" -> "cluster_initialize" [label="yes" color="green"]; + "data_belongs_to_cluster" -> "exit" [label="no" color="red"]; + "cluster_initialize" [label="Is cluster initialized on DCS?" shape="diamond"] + "cluster_initialize" -> "dcs.initialize" [label="no" color="red"] + "cluster_initialize" -> "is_healthy" [label="yes" color="green"] + "dcs.initialize" [label="Initialize new cluster"]; + "dcs.initialize" -> "is_healthy" + "is_healthy" [label="Is node healthy?\n(running Postgres)", shape="diamond"]; + "recover" [label="Restart as read-only\nand set Recover flag"] + "is_healthy" -> "recover" [label="no" color="red"]; + "is_healthy" -> "cluster.is_unlocked" [label="yes" color="green"]; + "cluster.is_unlocked" [label="Does the cluster have a leader?", shape="diamond"] + } + + "cluster.is_unlocked" -> "unhealthy_is_healthiest" [label="no" color="red"] + "cluster.is_unlocked" -> "healthy_has_lock" [label="yes" color="green"] + "bootstrap" -> "start" + "recover" -> "start" + "post_recover" -> "start"; + + subgraph cluster_process_healthy_cluster { + label = "process_healthy_cluster" + "healthy_has_lock" [label="Did I win the race to the leader lock?", shape=diamond] + "healthy_is_leader" [label="Is Postgres running as master?", shape=diamond] + "healthy_no_lock" [label="Is Postgres running as master?", shape=diamond] + "healthy_no_lock" -> "healthy_demote" [label="yes" color="green"] + "healthy_has_lock" -> "healthy_no_lock" [label="no" color="red"] + "healthy_has_lock" -> "healthy_is_leader" [label="yes" color="green"] + "healthy_failover" [label="Promote Postgres to master"] + "healthy_is_leader" -> "healthy_failover" [label="no" color="red"] + "healthy_is_leader" -> "renew_lock" [label="yes" color="green"] + "healthy_failover" -> "renew_lock" + "renew_lock" [label="Renew leader lock"] + "healthy_demote" [label="Demote Postgres to secondary"] + "healthy_no_lock" -> "healthy_start" [label="no" color="red"] + "healthy_demote" -> "healthy_start" + "renew_lock" -> "healthy_start" + "healthy_start" [label="Go to start", shape=rectangle, fillcolor=green] + } + + subgraph cluster_process_unhealthy_cluster { + label = "process_unhealthy_cluster" + "unhealthy_is_healthiest" [label="Am I the healthiest node?", shape="diamond"] + "unhealthy_is_healthiest" -> "unhealthy_acquire_lock" [label="yes, try to get lock"] + "unhealthy_acquire_lock" [label="Was I able to get the lock?", shape="diamond"] + "unhealthy_acquire_lock" -> "unhealthy_failover_key" [label="yes" color="green"] + "unhealthy_failover_key" [label="Is there a failover key set?", shape="diamond"] + "unhealthy_failover_key" -> "unhealthy_clean_failover_key" [label="yes" color="green"] + "unhealthy_failover_key" -> "unhealthy_promote" [label="yes" color="green"] + "unhealthy_clean_failover_key" [label="Clean failover key"] + "unhealthy_clean_failover_key" -> unhealthy_promote + "unhealthy_promote" [label="Promote to master"] + "unhealthy_is_healthiest" -> "unhealthy_need_rewind" [label="no" color="red"] + "unhealthy_need_rewind" [label="Do I need to be rewinded?", shape="diamond"] + "unhealthy_need_rewind" -> "unhealthy_set_rewind_flag" [label="yes" color="green"] + "unhealthy_need_rewind" -> "unhealthy_demote" [label="no" color="red"] + "unhealthy_set_rewind_flag" [label="Set rewind flag"] + unhealthy_set_rewind_flag -> unhealthy_demote + "unhealthy_demote" [label="Demote and follow somebody else"] + "unhealthy_demote" -> "unhealthy_start" + "unhealthy_acquire_lock" -> "unhealthy_start" [label="no" color="red"] + "unhealthy_promote" -> "unhealthy_start" + "unhealthy_start" [label="Go to start", shape=rectangle, fillcolor=green] + } + + "exit" [label="Fail and exit", fillcolor=red] +} diff --git a/docs/ha_loop_diagram.png b/docs/ha_loop_diagram.png new file mode 100644 index 00000000..0d5560b2 Binary files /dev/null and b/docs/ha_loop_diagram.png differ