From ac6f6ae1c2075b5f9e47b2be5bf9c2c4a9f46afd Mon Sep 17 00:00:00 2001 From: Ali Mehraji Date: Wed, 22 Nov 2023 11:25:51 +0330 Subject: [PATCH] Add ETCDCTL_API=3 env to Dockerfiles and update docker/README.md (#2946) --- Dockerfile | 1 + Dockerfile.citus | 1 + docker-compose-citus.yml | 2 - docker/README.md | 334 +++++++++++++++++++-------------------- 4 files changed, 169 insertions(+), 169 deletions(-) diff --git a/Dockerfile b/Dockerfile index c5b927ee..3e638518 100644 --- a/Dockerfile +++ b/Dockerfile @@ -143,6 +143,7 @@ ARG PGBIN=/usr/lib/postgresql/$PG_MAJOR/bin ENV LC_ALL=$LC_ALL LANG=$LANG EDITOR=/usr/bin/editor ENV PGDATA=$PGDATA PATH=$PATH:$PGBIN +ENV ETCDCTL_API=3 COPY patroni /patroni/ COPY extras/confd/conf.d/haproxy.toml /etc/confd/conf.d/ diff --git a/Dockerfile.citus b/Dockerfile.citus index 7e6ec18c..f52a36e7 100644 --- a/Dockerfile.citus +++ b/Dockerfile.citus @@ -164,6 +164,7 @@ ARG PGBIN=/usr/lib/postgresql/$PG_MAJOR/bin ENV LC_ALL=$LC_ALL LANG=$LANG EDITOR=/usr/bin/editor ENV PGDATA=$PGDATA PATH=$PATH:$PGBIN +ENV ETCDCTL_API=3 COPY patroni /patroni/ COPY extras/confd/conf.d/haproxy.toml /etc/confd/conf.d/ diff --git a/docker-compose-citus.yml b/docker-compose-citus.yml index 7ff2a2c5..cd63d583 100644 --- a/docker-compose-citus.yml +++ b/docker-compose-citus.yml @@ -19,7 +19,6 @@ services: image: ${PATRONI_TEST_IMAGE:-patroni-citus} networks: [ demo ] environment: - ETCDCTL_API: 3 ETCD_LISTEN_PEER_URLS: http://0.0.0.0:2380 ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379 ETCD_INITIAL_CLUSTER: etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380 @@ -53,7 +52,6 @@ services: - "5001:5001" # Load-balancing across workers primaries command: haproxy environment: &haproxy_env - ETCDCTL_API: 3 ETCDCTL_ENDPOINTS: http://etcd1:2379,http://etcd2:2379,http://etcd3:2379 PATRONI_ETCD3_HOSTS: "'etcd1:2379','etcd2:2379','etcd3:2379'" PATRONI_SCOPE: demo diff --git a/docker/README.md b/docker/README.md index da87842f..0b7f3d58 100644 --- a/docker/README.md +++ b/docker/README.md @@ -19,102 +19,97 @@ The haproxy listens on ports 5000 (connects to the primary) and 5001 (does load- Example session: - $ docker-compose up -d - Creating demo-haproxy ... - Creating demo-patroni2 ... - Creating demo-patroni1 ... - Creating demo-patroni3 ... - Creating demo-etcd2 ... - Creating demo-etcd1 ... - Creating demo-etcd3 ... - Creating demo-haproxy - Creating demo-patroni2 - Creating demo-patroni1 - Creating demo-patroni3 - Creating demo-etcd1 - Creating demo-etcd2 - Creating demo-etcd2 ... done + $ docker compose up -d + ✔ Network patroni_demo Created + ✔ Container demo-etcd1 Started + ✔ Container demo-haproxy Started + ✔ Container demo-patroni1 Started + ✔ Container demo-patroni2 Started + ✔ Container demo-patroni3 Started + ✔ Container demo-etcd2 Started + ✔ Container demo-etcd3 Started $ docker ps - CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - 5b7a90b4cfbf patroni "/bin/sh /entrypoint…" 29 seconds ago Up 27 seconds demo-etcd2 - e30eea5222f2 patroni "/bin/sh /entrypoint…" 29 seconds ago Up 27 seconds demo-etcd1 - 83bcf3cb208f patroni "/bin/sh /entrypoint…" 29 seconds ago Up 27 seconds demo-etcd3 - 922532c56e7d patroni "/bin/sh /entrypoint…" 29 seconds ago Up 28 seconds demo-patroni3 - 14f875e445f3 patroni "/bin/sh /entrypoint…" 29 seconds ago Up 28 seconds demo-patroni2 - 110d1073b383 patroni "/bin/sh /entrypoint…" 29 seconds ago Up 28 seconds demo-patroni1 - 5af5e6e36028 patroni "/bin/sh /entrypoint…" 29 seconds ago Up 28 seconds 0.0.0.0:5000-5001->5000-5001/tcp demo-haproxy + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + a37bcec56726 patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes demo-etcd3 + 034ab73868a8 patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes demo-patroni2 + 03837736f710 patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes demo-patroni3 + 22815c3d85b3 patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes demo-etcd2 + 814b4304d132 patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes 0.0.0.0:5000-5001->5000-5001/tcp, :::5000-5001->5000-5001/tcp demo-haproxy + 6375b0ba2d0a patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes demo-patroni1 + aef8bf3ee91f patroni "/bin/sh /entrypoint…" 15 minutes ago Up 15 minutes demo-etcd1 $ docker logs demo-patroni1 - 2019-02-20 08:19:32,714 INFO: Failed to import patroni.dcs.consul - 2019-02-20 08:19:32,737 INFO: Selected new etcd server http://etcd3:2379 - 2019-02-20 08:19:35,140 INFO: Lock owner: None; I am patroni1 - 2019-02-20 08:19:35,174 INFO: trying to bootstrap a new cluster + 2023-11-21 09:04:33,547 INFO: Selected new etcd server http://172.29.0.3:2379 + 2023-11-21 09:04:33,605 INFO: Lock owner: None; I am patroni1 + 2023-11-21 09:04:33,693 INFO: trying to bootstrap a new cluster ... - 2019-02-20 08:19:39,310 INFO: postmaster pid=37 - 2019-02-20 08:19:39.314 UTC [37] LOG: listening on IPv4 address "0.0.0.0", port 5432 - 2019-02-20 08:19:39.321 UTC [37] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432" - 2019-02-20 08:19:39.353 UTC [39] LOG: database system was shut down at 2019-02-20 08:19:36 UTC - 2019-02-20 08:19:39.354 UTC [40] FATAL: the database system is starting up - localhost:5432 - rejecting connections - 2019-02-20 08:19:39.369 UTC [37] LOG: database system is ready to accept connections + 2023-11-21 09:04:34.920 UTC [43] LOG: starting PostgreSQL 15.5 (Debian 15.5-1.pgdg120+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14) 12.2.0, 64-bit + 2023-11-21 09:04:34.921 UTC [43] LOG: listening on IPv4 address "0.0.0.0", port 5432 + 2023-11-21 09:04:34,922 INFO: postmaster pid=43 + 2023-11-21 09:04:34.922 UTC [43] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432" + 2023-11-21 09:04:34.925 UTC [47] LOG: database system was shut down at 2023-11-21 09:04:34 UTC + 2023-11-21 09:04:34.928 UTC [43] LOG: database system is ready to accept connections localhost:5432 - accepting connections - 2019-02-20 08:19:39,383 INFO: establishing a new patroni connection to the postgres cluster - 2019-02-20 08:19:39,408 INFO: running post_bootstrap - 2019-02-20 08:19:39,432 WARNING: Could not activate Linux watchdog device: "Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog'" - 2019-02-20 08:19:39,515 INFO: initialized a new cluster - 2019-02-20 08:19:49,424 INFO: Lock owner: patroni1; I am patroni1 - 2019-02-20 08:19:49,447 INFO: Lock owner: patroni1; I am patroni1 - 2019-02-20 08:19:49,480 INFO: no action. i am the leader with the lock - 2019-02-20 08:19:59,422 INFO: Lock owner: patroni1; I am patroni1 + localhost:5432 - accepting connections + 2023-11-21 09:04:34,938 INFO: establishing a new patroni heartbeat connection to postgres + 2023-11-21 09:04:34,992 INFO: running post_bootstrap + 2023-11-21 09:04:35,004 WARNING: User creation via "bootstrap.users" will be removed in v4.0.0 + 2023-11-21 09:04:35,009 WARNING: Could not activate Linux watchdog device: Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog' + 2023-11-21 09:04:35,189 INFO: initialized a new cluster + 2023-11-21 09:04:35,328 INFO: no action. I am (patroni1), the leader with the lock + 2023-11-21 09:04:43,824 INFO: establishing a new patroni restapi connection to postgres + 2023-11-21 09:04:45,322 INFO: no action. I am (patroni1), the leader with the lock + 2023-11-21 09:04:55,320 INFO: no action. I am (patroni1), the leader with the lock + ... $ docker exec -ti demo-patroni1 bash postgres@patroni1:~$ patronictl list - +---------+----------+------------+--------+---------+----+-----------+ - | Cluster | Member | Host | Role | State | TL | Lag in MB | - +---------+----------+------------+--------+---------+----+-----------+ - | demo | patroni1 | 172.22.0.3 | Leader | running | 1 | 0 | - | demo | patroni2 | 172.22.0.7 | | running | 1 | 0 | - | demo | patroni3 | 172.22.0.4 | | running | 1 | 0 | - +---------+----------+------------+--------+---------+----+-----------+ + + Cluster: demo (7303838734793224214) --------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +----------+------------+---------+-----------+----+-----------+ + | patroni1 | 172.29.0.2 | Leader | running | 1 | | + | patroni2 | 172.29.0.6 | Replica | streaming | 1 | 0 | + | patroni3 | 172.29.0.5 | Replica | streaming | 1 | 0 | + +----------+------------+---------+-----------+----+-----------+ postgres@patroni1:~$ etcdctl get --keys-only --prefix /service/demo /service/demo/config /service/demo/initialize /service/demo/leader - /service/demo/members/ /service/demo/members/patroni1 /service/demo/members/patroni2 /service/demo/members/patroni3 - /service/demo/optime/ - /service/demo/optime/leader + /service/demo/status postgres@patroni1:~$ etcdctl member list - 1bab629f01fa9065: name=etcd3 peerURLs=http://etcd3:2380 clientURLs=http://etcd3:2379 isLeader=false - 8ecb6af518d241cc: name=etcd2 peerURLs=http://etcd2:2380 clientURLs=http://etcd2:2379 isLeader=true - b2e169fcb8a34028: name=etcd1 peerURLs=http://etcd1:2380 clientURLs=http://etcd1:2379 isLeader=false + 2bf3e2ceda5d5960, started, etcd2, http://etcd2:2380, http://172.29.0.3:2379 + 55b3264e129c7005, started, etcd3, http://etcd3:2380, http://172.29.0.7:2379 + acce7233f8ec127e, started, etcd1, http://etcd1:2380, http://172.29.0.8:2379 + + postgres@patroni1:~$ exit $ docker exec -ti demo-haproxy bash postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -W Password: postgres - psql (11.2 (Ubuntu 11.2-1.pgdg18.04+1), server 10.7 (Debian 10.7-1.pgdg90+1)) + psql (15.5 (Debian 15.5-1.pgdg120+1)) Type "help" for help. - localhost/postgres=# select pg_is_in_recovery(); + postgres=# SELECT pg_is_in_recovery(); pg_is_in_recovery ─────────────────── f (1 row) - localhost/postgres=# \q + postgres=# \q - $postgres@haproxy:~ psql -h localhost -p 5001 -U postgres -W + postgres@haproxy:~$ psql -h localhost -p 5001 -U postgres -W Password: postgres - psql (11.2 (Ubuntu 11.2-1.pgdg18.04+1), server 10.7 (Debian 10.7-1.pgdg90+1)) + psql (15.5 (Debian 15.5-1.pgdg120+1)) Type "help" for help. - localhost/postgres=# select pg_is_in_recovery(); + postgres=# SELECT pg_is_in_recovery(); pg_is_in_recovery ─────────────────── t @@ -127,81 +122,86 @@ The haproxy listens on ports 5000 (connects to the coordinator primary) and 5001 Example session: - $ docker-compose -f docker-compose-citus.yml up -d - Creating demo-work2-1 ... done - Creating demo-work1-1 ... done - Creating demo-etcd2 ... done - Creating demo-etcd1 ... done - Creating demo-coord3 ... done - Creating demo-etcd3 ... done - Creating demo-coord1 ... done - Creating demo-haproxy ... done - Creating demo-work2-2 ... done - Creating demo-coord2 ... done - Creating demo-work1-2 ... done + $ docker compose -f docker-compose-citus.yml up -d + ✔ Network patroni_demo Created + ✔ Container demo-coord2 Started + ✔ Container demo-work2-2 Started + ✔ Container demo-etcd1 Started + ✔ Container demo-haproxy Started + ✔ Container demo-work1-1 Started + ✔ Container demo-work2-1 Started + ✔ Container demo-work1-2 Started + ✔ Container demo-coord1 Started + ✔ Container demo-etcd3 Started + ✔ Container demo-coord3 Started + ✔ Container demo-etcd2 Started + $ docker ps - CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - 852d8885a612 patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 3 seconds demo-coord3 - cdd692f947ab patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 3 seconds demo-work1-2 - 9f4e340b36da patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 3 seconds demo-etcd3 - d69c129a960a patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 4 seconds demo-etcd1 - c5849689b8cd patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 4 seconds demo-coord1 - c9d72bd6217d patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 3 seconds demo-work2-1 - 24b1b43efa05 patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 4 seconds demo-coord2 - cb0cc2b4ca0a patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 3 seconds demo-work2-2 - 9796c6b8aad5 patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 5 seconds demo-work1-1 - 8baccd74dcae patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 4 seconds demo-etcd2 - 353ec62a0187 patroni-citus "/bin/sh /entrypoint…" 6 seconds ago Up 4 seconds 0.0.0.0:5000-5001->5000-5001/tcp demo-haproxy + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 79c95492fac9 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-etcd3 + 77eb82d0f0c1 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-work2-1 + 03dacd7267ef patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-etcd1 + db9206c66f85 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-etcd2 + 9a0fef7b7dd4 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-work1-2 + f06b031d99dc patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-work2-2 + f7c58545f314 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-coord2 + 383f9e7e188a patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-work1-1 + f02e96dcc9d6 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-coord3 + 6945834b7056 patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes demo-coord1 + b96ca42f785d patroni-citus "/bin/sh /entrypoint…" 11 minutes ago Up 11 minutes 0.0.0.0:5000-5001->5000-5001/tcp, :::5000-5001->5000-5001/tcp demo-haproxy + $ docker logs demo-coord1 - 2023-01-05 15:09:31,295 INFO: Selected new etcd server http://172.27.0.4:2379 - 2023-01-05 15:09:31,388 INFO: Lock owner: None; I am coord1 - 2023-01-05 15:09:31,501 INFO: trying to bootstrap a new cluster + 2023-11-21 09:36:14,293 INFO: Selected new etcd server http://172.30.0.4:2379 + 2023-11-21 09:36:14,390 INFO: Lock owner: None; I am coord1 + 2023-11-21 09:36:14,478 INFO: trying to bootstrap a new cluster ... - 2023-01-05 15:09:45,096 INFO: postmaster pid=39 + 2023-11-21 09:36:16,475 INFO: postmaster pid=52 localhost:5432 - no response - 2023-01-05 15:09:45.137 UTC [39] LOG: starting PostgreSQL 15.1 (Debian 15.1-1.pgdg110+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 10.2.1-6) 10.2.1 20210110, 64-bit - 2023-01-05 15:09:45.137 UTC [39] LOG: listening on IPv4 address "0.0.0.0", port 5432 - 2023-01-05 15:09:45.152 UTC [39] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432" - 2023-01-05 15:09:45.177 UTC [43] LOG: database system was shut down at 2023-01-05 15:09:32 UTC - 2023-01-05 15:09:45.193 UTC [39] LOG: database system is ready to accept connections + 2023-11-21 09:36:16.495 UTC [52] LOG: starting PostgreSQL 15.5 (Debian 15.5-1.pgdg120+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14) 12.2.0, 64-bit + 2023-11-21 09:36:16.495 UTC [52] LOG: listening on IPv4 address "0.0.0.0", port 5432 + 2023-11-21 09:36:16.496 UTC [52] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432" + 2023-11-21 09:36:16.498 UTC [56] LOG: database system was shut down at 2023-11-21 09:36:15 UTC + 2023-11-21 09:36:16.501 UTC [52] LOG: database system is ready to accept connections localhost:5432 - accepting connections localhost:5432 - accepting connections - 2023-01-05 15:09:46,139 INFO: establishing a new patroni connection to the postgres cluster - 2023-01-05 15:09:46,208 INFO: running post_bootstrap - 2023-01-05 15:09:47.209 UTC [55] LOG: starting maintenance daemon on database 16386 user 10 - 2023-01-05 15:09:47.209 UTC [55] CONTEXT: Citus maintenance daemon for database 16386 user 10 - 2023-01-05 15:09:47,215 WARNING: Could not activate Linux watchdog device: "Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog'" - 2023-01-05 15:09:47.446 UTC [41] LOG: checkpoint starting: immediate force wait - 2023-01-05 15:09:47,466 INFO: initialized a new cluster - 2023-01-05 15:09:47,594 DEBUG: query(SELECT nodeid, groupid, nodename, nodeport, noderole FROM pg_catalog.pg_dist_node WHERE noderole = 'primary', ()) - 2023-01-05 15:09:47,594 INFO: establishing a new patroni connection to the postgres cluster - 2023-01-05 15:09:47,467 INFO: Lock owner: coord1; I am coord1 - 2023-01-05 15:09:47,613 DEBUG: query(SELECT pg_catalog.citus_set_coordinator_host(%s, %s, 'primary', 'default'), ('172.27.0.6', 5432)) - 2023-01-05 15:09:47,924 INFO: no action. I am (coord1), the leader with the lock - 2023-01-05 15:09:51.282 UTC [41] LOG: checkpoint complete: wrote 1086 buffers (53.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.029 s, sync=3.746 s, total=3.837 s; sync files=280, longest=0.028 s, average=0.014 s; distance=8965 kB, estimate=8965 kB - 2023-01-05 15:09:51.283 UTC [41] LOG: checkpoint starting: immediate force wait - 2023-01-05 15:09:51.495 UTC [41] LOG: checkpoint complete: wrote 18 buffers (0.9%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.044 s, sync=0.091 s, total=0.212 s; sync files=15, longest=0.015 s, average=0.007 s; distance=67 kB, estimate=8076 kB - 2023-01-05 15:09:57,467 INFO: Lock owner: coord1; I am coord1 - 2023-01-05 15:09:57,569 INFO: Assigning synchronous standby status to ['coord3'] + 2023-11-21 09:36:17,509 INFO: establishing a new patroni heartbeat connection to postgres + 2023-11-21 09:36:17,569 INFO: running post_bootstrap + 2023-11-21 09:36:17,593 WARNING: User creation via "bootstrap.users" will be removed in v4.0.0 + 2023-11-21 09:36:17,783 INFO: establishing a new patroni restapi connection to postgres + 2023-11-21 09:36:17,969 WARNING: Could not activate Linux watchdog device: Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog' + 2023-11-21 09:36:17.969 UTC [70] LOG: starting maintenance daemon on database 16386 user 10 + 2023-11-21 09:36:17.969 UTC [70] CONTEXT: Citus maintenance daemon for database 16386 user 10 + 2023-11-21 09:36:18.159 UTC [54] LOG: checkpoint starting: immediate force wait + 2023-11-21 09:36:18,162 INFO: initialized a new cluster + 2023-11-21 09:36:18,164 INFO: Lock owner: coord1; I am coord1 + 2023-11-21 09:36:18,297 INFO: Enabled synchronous replication + 2023-11-21 09:36:18,298 DEBUG: Adding the new task: PgDistNode(nodeid=None,group=0,host=172.30.0.3,port=5432,event=after_promote) + 2023-11-21 09:36:18,298 DEBUG: Adding the new task: PgDistNode(nodeid=None,group=1,host=172.30.0.7,port=5432,event=after_promote) + 2023-11-21 09:36:18,298 DEBUG: Adding the new task: PgDistNode(nodeid=None,group=2,host=172.30.0.8,port=5432,event=after_promote) + 2023-11-21 09:36:18,299 DEBUG: query(SELECT nodeid, groupid, nodename, nodeport, noderole FROM pg_catalog.pg_dist_node WHERE noderole = 'primary', ()) + 2023-11-21 09:36:18,299 INFO: establishing a new patroni citus connection to postgres + 2023-11-21 09:36:18,323 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.30.0.7', 5432, 1)) + 2023-11-21 09:36:18,361 INFO: no action. I am (coord1), the leader with the lock + 2023-11-21 09:36:18,393 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.30.0.8', 5432, 2)) + 2023-11-21 09:36:28,164 INFO: Lock owner: coord1; I am coord1 + 2023-11-21 09:36:28,251 INFO: Assigning synchronous standby status to ['coord3'] server signaled - 2023-01-05 15:09:57.574 UTC [39] LOG: received SIGHUP, reloading configuration files - 2023-01-05 15:09:57.580 UTC [39] LOG: parameter "synchronous_standby_names" changed to "coord3" - 2023-01-05 15:09:59,637 INFO: Synchronous standby status assigned to ['coord3'] - 2023-01-05 15:09:59,638 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.27.0.2', 5432, 1)) - 2023-01-05 15:09:59.690 UTC [67] LOG: standby "coord3" is now a synchronous standby with priority 1 - 2023-01-05 15:09:59.690 UTC [67] STATEMENT: START_REPLICATION SLOT "coord3" 0/3000000 TIMELINE 1 - 2023-01-05 15:09:59,694 INFO: no action. I am (coord1), the leader with the lock - 2023-01-05 15:09:59,704 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.27.0.8', 5432, 2)) - 2023-01-05 15:10:07,625 INFO: no action. I am (coord1), the leader with the lock - 2023-01-05 15:10:17,579 INFO: no action. I am (coord1), the leader with the lock + 2023-11-21 09:36:28.435 UTC [52] LOG: received SIGHUP, reloading configuration files + 2023-11-21 09:36:28.436 UTC [52] LOG: parameter "synchronous_standby_names" changed to "coord3" + 2023-11-21 09:36:28.641 UTC [83] LOG: standby "coord3" is now a synchronous standby with priority 1 + 2023-11-21 09:36:28.641 UTC [83] STATEMENT: START_REPLICATION SLOT "coord3" 0/3000000 TIMELINE 1 + 2023-11-21 09:36:30,582 INFO: Synchronous standby status assigned to ['coord3'] + 2023-11-21 09:36:30,626 INFO: no action. I am (coord1), the leader with the lock + 2023-11-21 09:36:38,250 INFO: no action. I am (coord1), the leader with the lock + ... $ docker exec -ti demo-haproxy bash postgres@haproxy:~$ etcdctl member list - 1bab629f01fa9065, started, etcd3, http://etcd3:2380, http://172.27.0.10:2379 - 8ecb6af518d241cc, started, etcd2, http://etcd2:2380, http://172.27.0.4:2379 - b2e169fcb8a34028, started, etcd1, http://etcd1:2380, http://172.27.0.7:2379 + 2b28411e74c0c281, started, etcd3, http://etcd3:2380, http://172.30.0.4:2379 + 6c70137d27cfa6c1, started, etcd2, http://etcd2:2380, http://172.30.0.5:2379 + a28f9a70ebf21304, started, etcd1, http://etcd1:2380, http://172.30.0.6:2379 postgres@haproxy:~$ etcdctl get --keys-only --prefix /service/demo /service/demo/0/config @@ -229,7 +229,7 @@ Example session: postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -d citus Password for user postgres: postgres - psql (15.1 (Debian 15.1-1.pgdg110+1)) + psql (15.5 (Debian 15.5-1.pgdg120+1)) SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, compression: off) Type "help" for help. @@ -240,67 +240,67 @@ Example session: (1 row) citus=# table pg_dist_node; - nodeid | groupid | nodename | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards + nodeid | groupid | nodename | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards --------+---------+------------+----------+----------+-------------+----------+----------+-------------+----------------+------------------ - 1 | 0 | 172.27.0.6 | 5432 | default | t | t | primary | default | t | f - 2 | 1 | 172.27.0.2 | 5432 | default | t | t | primary | default | t | t - 3 | 2 | 172.27.0.8 | 5432 | default | t | t | primary | default | t | t + 1 | 0 | 172.30.0.3 | 5432 | default | t | t | primary | default | t | f + 2 | 1 | 172.30.0.7 | 5432 | default | t | t | primary | default | t | t + 3 | 2 | 172.30.0.8 | 5432 | default | t | t | primary | default | t | t (3 rows) citus=# \q postgres@haproxy:~$ patronictl list - + Citus cluster: demo ----------+--------------+---------+----+-----------+ - | Group | Member | Host | Role | State | TL | Lag in MB | - +-------+---------+-------------+--------------+---------+----+-----------+ - | 0 | coord1 | 172.27.0.6 | Leader | running | 1 | | - | 0 | coord2 | 172.27.0.5 | Replica | running | 1 | 0 | - | 0 | coord3 | 172.27.0.9 | Sync Standby | running | 1 | 0 | - | 1 | work1-1 | 172.27.0.2 | Leader | running | 1 | | - | 1 | work1-2 | 172.27.0.12 | Sync Standby | running | 1 | 0 | - | 2 | work2-1 | 172.27.0.11 | Sync Standby | running | 1 | 0 | - | 2 | work2-2 | 172.27.0.8 | Leader | running | 1 | | - +-------+---------+-------------+--------------+---------+----+-----------+ + + Citus cluster: demo ----------+--------------+-----------+----+-----------+ + | Group | Member | Host | Role | State | TL | Lag in MB | + +-------+---------+-------------+--------------+-----------+----+-----------+ + | 0 | coord1 | 172.30.0.3 | Leader | running | 1 | | + | 0 | coord2 | 172.30.0.12 | Replica | streaming | 1 | 0 | + | 0 | coord3 | 172.30.0.2 | Sync Standby | streaming | 1 | 0 | + | 1 | work1-1 | 172.30.0.7 | Leader | running | 1 | | + | 1 | work1-2 | 172.30.0.10 | Sync Standby | streaming | 1 | 0 | + | 2 | work2-1 | 172.30.0.8 | Leader | running | 1 | | + | 2 | work2-2 | 172.30.0.11 | Sync Standby | streaming | 1 | 0 | + +-------+---------+-------------+--------------+-----------+----+-----------+ + postgres@haproxy:~$ patronictl switchover --group 2 --force Current cluster topology - + Citus cluster: demo (group: 2, 7185185529556963355) +-----------+ - | Member | Host | Role | State | TL | Lag in MB | - +---------+-------------+--------------+---------+----+-----------+ - | work2-1 | 172.27.0.11 | Sync Standby | running | 1 | 0 | - | work2-2 | 172.27.0.8 | Leader | running | 1 | | - +---------+-------------+--------------+---------+----+-----------+ - 2023-01-05 15:29:29.54204 Successfully switched over to "work2-1" - + Citus cluster: demo (group: 2, 7185185529556963355) -------+ + + Citus cluster: demo (group: 2, 7303846899271086103) --+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +---------+-------------+--------------+-----------+----+-----------+ + | work2-1 | 172.30.0.8 | Leader | running | 1 | | + | work2-2 | 172.30.0.11 | Sync Standby | streaming | 1 | 0 | + +---------+-------------+--------------+-----------+----+-----------+ + 2023-11-21 09:44:15.83849 Successfully switched over to "work2-2" + + Citus cluster: demo (group: 2, 7303846899271086103) -------+ | Member | Host | Role | State | TL | Lag in MB | +---------+-------------+---------+---------+----+-----------+ - | work2-1 | 172.27.0.11 | Leader | running | 1 | | - | work2-2 | 172.27.0.8 | Replica | stopped | | unknown | + | work2-1 | 172.30.0.8 | Replica | stopped | | unknown | + | work2-2 | 172.30.0.11 | Leader | running | 1 | | +---------+-------------+---------+---------+----+-----------+ postgres@haproxy:~$ patronictl list - + Citus cluster: demo ----------+--------------+---------+----+-----------+ - | Group | Member | Host | Role | State | TL | Lag in MB | - +-------+---------+-------------+--------------+---------+----+-----------+ - | 0 | coord1 | 172.27.0.6 | Leader | running | 1 | | - | 0 | coord2 | 172.27.0.5 | Replica | running | 1 | 0 | - | 0 | coord3 | 172.27.0.9 | Sync Standby | running | 1 | 0 | - | 1 | work1-1 | 172.27.0.2 | Leader | running | 1 | | - | 1 | work1-2 | 172.27.0.12 | Sync Standby | running | 1 | 0 | - | 2 | work2-1 | 172.27.0.11 | Leader | running | 2 | | - | 2 | work2-2 | 172.27.0.8 | Sync Standby | running | 2 | 0 | - +-------+---------+-------------+--------------+---------+----+-----------+ + + Citus cluster: demo ----------+--------------+-----------+----+-----------+ + | Group | Member | Host | Role | State | TL | Lag in MB | + +-------+---------+-------------+--------------+-----------+----+-----------+ + | 0 | coord1 | 172.30.0.3 | Leader | running | 1 | | + | 0 | coord2 | 172.30.0.12 | Replica | streaming | 1 | 0 | + | 0 | coord3 | 172.30.0.2 | Sync Standby | streaming | 1 | 0 | + | 1 | work1-1 | 172.30.0.7 | Leader | running | 1 | | + | 1 | work1-2 | 172.30.0.10 | Sync Standby | streaming | 1 | 0 | + | 2 | work2-1 | 172.30.0.8 | Sync Standby | streaming | 2 | 0 | + | 2 | work2-2 | 172.30.0.11 | Leader | running | 2 | | + +-------+---------+-------------+--------------+-----------+----+-----------+ postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -d citus - Password for user postgres: postgres - psql (15.1 (Debian 15.1-1.pgdg110+1)) + psql (15.5 (Debian 15.5-1.pgdg120+1)) SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, compression: off) Type "help" for help. citus=# table pg_dist_node; - nodeid | groupid | nodename | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards + nodeid | groupid | nodename | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards --------+---------+-------------+----------+----------+-------------+----------+----------+-------------+----------------+------------------ - 1 | 0 | 172.27.0.6 | 5432 | default | t | t | primary | default | t | f - 3 | 2 | 172.27.0.11 | 5432 | default | t | t | primary | default | t | t - 2 | 1 | 172.27.0.2 | 5432 | default | t | t | primary | default | t | t + 1 | 0 | 172.30.0.3 | 5432 | default | t | t | primary | default | t | f + 3 | 2 | 172.30.0.11 | 5432 | default | t | t | primary | default | t | t + 2 | 1 | 172.30.0.7 | 5432 | default | t | t | primary | default | t | t (3 rows)