From 61e01b1435bbd6aaf021b9bc5c0f741d958f90c8 Mon Sep 17 00:00:00 2001 From: Feike Steenbergen Date: Tue, 1 Sep 2015 11:35:32 +0200 Subject: [PATCH 1/3] Update documentation and Dockerfile to point to a SNAPSHOT version. Include python-etcd in Dockerfile --- Dockerfile | 3 ++- docker/README.md | 14 +++++++------- docker/dev_patroni_cluster.sh | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index d004e4b2..2068d856 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,8 @@ RUN apt-get update -y RUN apt-get upgrade -y ENV PGVERSION 9.4 -RUN apt-get install python python-psycopg2 python-yaml python-requests python-boto postgresql-${PGVERSION} python-dnspython python-kazoo -y +RUN apt-get install python python-psycopg2 python-yaml python-requests python-boto postgresql-${PGVERSION} python-dnspython python-kazoo python-pip -y +RUN pip install python-etcd ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH diff --git a/docker/README.md b/docker/README.md index e6245adc..d17e77a1 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,6 +1,6 @@ # Patroni Dockerfile You can run Patroni in a docker container using this Dockerfile, or by using the Docker image at - https://os-registry.stups.zalan.do/acid/patroni-1.0 + https://os-registry.stups.zalan.do/acid/patroni-1.0-SNAPSHOT This Dockerfile is meant in aiding development of Patroni and quick testing of features. It is not a production-worthy Dockerfile @@ -9,7 +9,7 @@ Dockerfile ## Standalone Patroni - docker run -d os-registry.stups.zalan.do/acid/patroni:1.0 + docker run -d os-registry.stups.zalan.do/acid/patroni:1.0-SNAPSHOT ## Multiple Patroni's communicating with a standalone etcd inside Docker @@ -35,12 +35,12 @@ To automate this you can run the following script: Example session: - $ ./dev_patroni_cluster.sh --image os-registry.stups.zalan.do/acid/patroni:1.0 --members=2 --name=bravo - The etcd container is 6be871a11cb373406ca5ea1c6b39e140fdde9fb1d6177212d6ad0c0d1bd9b563, ip=172.17.1.24 + $ ./dev_patroni_cluster.sh --image os-registry.stups.zalan.do/acid/patroni:1.0-SNAPSHOT --members=2 --name=bravo + The etcd container is 6be871a11cb373406ca5ea1c6b39e1.0-SNAPSHOTfdde9fb1d6177212d6ad0c0d1bd9b563, ip=172.17.1.24 Started Patroni container 67e611f2eca7c40f9e6e0e24a4a8f2cba7e3e56d22a420e15ab9240a37a9d7a4, ip=172.17.1.25 Started Patroni container 47dd12ae635ab83b039f5889e250048b606ed5e48e3650b69e365e7e1d4acbcf, ip=172.17.1.26 $ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - 47dd12ae635a os-registry.stups.zalan.do/acid/patroni:1.0 "/bin/bash /entrypoi 10 seconds ago Up 8 seconds 4001/tcp, 5432/tcp, 2380/tcp bravo_OR64g8bx - 67e611f2eca7 os-registry.stups.zalan.do/acid/patroni:1.0 "/bin/bash /entrypoi 11 seconds ago Up 10 seconds 2380/tcp, 4001/tcp, 5432/tcp bravo_si9no8iz - 6be871a11cb3 os-registry.stups.zalan.do/acid/patroni:1.0 "/bin/bash /entrypoi 12 seconds ago Up 10 seconds 4001/tcp, 5432/tcp, 2380/tcp bravo_etcd + 47dd12ae635a os-registry.stups.zalan.do/acid/patroni:1.0-SNAPSHOT "/bin/bash /entrypoi 10 seconds ago Up 8 seconds 4001/tcp, 5432/tcp, 2380/tcp bravo_OR64g8bx + 67e611f2eca7 os-registry.stups.zalan.do/acid/patroni:1.0-SNAPSHOT "/bin/bash /entrypoi 11 seconds ago Up 10 seconds 2380/tcp, 4001/tcp, 5432/tcp bravo_si9no8iz + 6be871a11cb3 os-registry.stups.zalan.do/acid/patroni:1.0-SNAPSHOT "/bin/bash /entrypoi 12 seconds ago Up 10 seconds 4001/tcp, 5432/tcp, 2380/tcp bravo_etcd diff --git a/docker/dev_patroni_cluster.sh b/docker/dev_patroni_cluster.sh index c77a8fb3..e9a253dc 100755 --- a/docker/dev_patroni_cluster.sh +++ b/docker/dev_patroni_cluster.sh @@ -1,6 +1,6 @@ #!/bin/bash -DOCKER_IMAGE="os-registry.stups.zalan.do/acid/patroni:1.0" +DOCKER_IMAGE="os-registry.stups.zalan.do/acid/patroni:1.0-SNAPSHOT" MEMBERS=3 From 3e4d2df33bb4f78531da0a79496c89fe5f4ad83c Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Wed, 2 Sep 2015 12:04:39 +0200 Subject: [PATCH 2/3] put a real restore/replication command inside the test postgres.yaml files --- postgres0.yml | 2 +- postgres1.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/postgres0.yml b/postgres0.yml index a2a7ce44..dd8f8356 100644 --- a/postgres0.yml +++ b/postgres0.yml @@ -46,7 +46,7 @@ postgresql: env_dir: /home/postgres/etc/wal-e.d/env threshold_megabytes: 10240 threshold_backup_size_percentage: 30 - restore: "true" + restore: scripts/restore.py #recovery_conf: #restore_command: cp ../wal_archive/%f %p parameters: diff --git a/postgres1.yml b/postgres1.yml index 6ef6b1c9..c2cb5ee6 100644 --- a/postgres1.yml +++ b/postgres1.yml @@ -48,6 +48,7 @@ postgresql: env_dir: /home/postgres/etc/wal-e.d/env threshold_megabytes: 10240 threshold_backup_size_percentage: 30 + restore: scripts/restore.py parameters: archive_mode: "on" wal_level: hot_standby From a5f2ab6f88bff36d0f056478a874467af938a830 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Wed, 2 Sep 2015 12:56:23 +0200 Subject: [PATCH 3/3] Better SIGCHLD handling, use correct user name. - Defer SIGCHLD processing until the later stage in order to allow subprocess to run its own reaping of children and return the correct exit code from the process called. - When changing the password at startup, do not assume the user postgres. --- helpers/postgresql.py | 2 +- helpers/utils.py | 37 +++++++++++++++++++++++-------------- patroni.py | 3 ++- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/helpers/postgresql.py b/helpers/postgresql.py index ce8ca18f..695e45c1 100644 --- a/helpers/postgresql.py +++ b/helpers/postgresql.py @@ -354,7 +354,7 @@ primary_conninfo = '{}' self.query('CREATE ROLE "{0}" WITH LOGIN SUPERUSER PASSWORD %s'.format( self.superuser['username']), self.superuser['password']) else: - self.query('ALTER ROLE postgres WITH PASSWORD %s', self.superuser['password']) + self.query('ALTER ROLE "{0}" WITH PASSWORD %s'.format(os.environ['USER']), self.superuser['password']) if self.admin: self.query('CREATE ROLE "{0}" WITH LOGIN CREATEDB CREATEROLE PASSWORD %s'.format( self.admin['username']), self.admin['password']) diff --git a/helpers/utils.py b/helpers/utils.py index 5f09490e..db578de5 100644 --- a/helpers/utils.py +++ b/helpers/utils.py @@ -5,7 +5,8 @@ import signal import sys import time -received_sigchld = False +interrupted_sleep = False +reap_children = False _DATE_TIME_RE = re.compile(r'''^ (?P\d{4})\-(?P\d{2})\-(?P\d{2}) # date @@ -72,30 +73,38 @@ def sigterm_handler(signo, stack_frame): def sigchld_handler(signo, stack_frame): - global received_sigchld - received_sigchld = True - try: - while True: - ret = os.waitpid(-1, os.WNOHANG) - if ret == (0, 0): - break - except OSError: - pass + global interrupted_sleep, reap_children + reap_children = interrupted_sleep = True def sleep(interval): - global received_sigchld + global interrupted_sleep current_time = time.time() end_time = current_time + interval while current_time < end_time: - received_sigchld = False + interrupted_sleep = False time.sleep(end_time - current_time) - if not received_sigchld: # we will ignore only sigchld + if not interrupted_sleep: # we will ignore only sigchld break current_time = time.time() - received_sigchld = False + interrupted_sleep = False def setup_signal_handlers(): signal.signal(signal.SIGTERM, sigterm_handler) signal.signal(signal.SIGCHLD, sigchld_handler) + + +def reap_children(): + global reap_children + if reap_children: + try: + while True: + ret = os.waitpid(-1, os.WNOHANG) + print ret + if ret == (0, 0): + break + except OSError: + pass + finally: + reap_children = False diff --git a/patroni.py b/patroni.py index e26a4059..a15f535b 100755 --- a/patroni.py +++ b/patroni.py @@ -9,7 +9,7 @@ from helpers.api import RestApiServer from helpers.etcd import Etcd from helpers.ha import Ha from helpers.postgresql import Postgresql -from helpers.utils import setup_signal_handlers, sleep +from helpers.utils import setup_signal_handlers, sleep, reap_children from helpers.zookeeper import ZooKeeper logger = logging.getLogger(__name__) @@ -93,6 +93,7 @@ class Patroni: except: logger.exception('Exception when changing replication slots') self.schedule_next_run() + reap_children() def main():