From 2174d66f976d65b396cc83c605c762d9be809673 Mon Sep 17 00:00:00 2001 From: Igor Yanchenko <1504692+yanchenko-igor@users.noreply.github.com> Date: Wed, 11 Dec 2019 12:07:05 +0100 Subject: [PATCH] Rewriten shell scripts in python to make them compatible with windows (#1326) --- features/archive-restore.py | 21 +++++++++++++++++++++ features/backup_create.py | 14 ++++++++++++++ features/backup_create.sh | 22 ---------------------- features/backup_restore.py | 11 +++++++++++ features/backup_restore.sh | 21 --------------------- features/callback.py | 17 +++++++++++++++++ features/callback.sh | 5 ----- features/callback2.py | 5 +++++ features/environment.py | 24 ++++++++++++++---------- features/standby_cluster.feature | 2 +- features/steps/cascading_replication.py | 2 +- features/steps/standby_cluster.py | 9 +++++---- 12 files changed, 89 insertions(+), 64 deletions(-) create mode 100644 features/archive-restore.py create mode 100755 features/backup_create.py delete mode 100755 features/backup_create.sh create mode 100755 features/backup_restore.py delete mode 100755 features/backup_restore.sh create mode 100755 features/callback.py delete mode 100755 features/callback.sh create mode 100755 features/callback2.py diff --git a/features/archive-restore.py b/features/archive-restore.py new file mode 100644 index 00000000..91c216db --- /dev/null +++ b/features/archive-restore.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +import os +import argparse +import shutil + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--dirname", required=True) + parser.add_argument("--pathname", required=True) + parser.add_argument("--filename", required=True) + parser.add_argument("--mode", required=True, choices=("archive", "restore")) + args, _ = parser.parse_known_args() + + full_filename = os.path.join(args.dirname, args.filename) + if args.mode == "archive": + if not os.path.isdir(args.dirname): + os.makedirs(args.dirname) + if not os.path.exists(full_filename): + shutil.copy(args.pathname, full_filename) + else: + shutil.copy(full_filename, args.pathname) diff --git a/features/backup_create.py b/features/backup_create.py new file mode 100755 index 00000000..65ce52a6 --- /dev/null +++ b/features/backup_create.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +import argparse +import subprocess +import sys + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--datadir", required=True) + parser.add_argument("--dbname", required=True) + parser.add_argument("--walmethod", required=True, choices=("fetch", "stream", "none")) + args, _ = parser.parse_known_args() + + walmethod = ["-X", args.walmethod] if args.walmethod != "none" else [] + sys.exit(subprocess.call(["pg_basebackup", "-D", args.datadir, "-c", "fast", "-d", args.dbname] + walmethod)) diff --git a/features/backup_create.sh b/features/backup_create.sh deleted file mode 100755 index ad39e8ea..00000000 --- a/features/backup_create.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -while getopts ":-:" optchar; do - [[ "${optchar}" == "-" ]] || continue - case "${OPTARG}" in - datadir=* ) - PGDATA=${OPTARG#*=} - ;; - dbname=* ) - DBNAME=${OPTARG#*=} - ;; - walmethod=* ) - WALMETHOD=${OPTARG#*=} - ;; - esac -done - -[[ -z $PGDATA || -z $DBNAME || -z $WALMETHOD ]] && exit 1 - -[[ $WALMETHOD != "none" ]] && WALMETHOD="-X $WALMETHOD" || WALMETHOD="" - -exec pg_basebackup -D $PGDATA $WALMETHOD -c fast -d $DBNAME diff --git a/features/backup_restore.py b/features/backup_restore.py new file mode 100755 index 00000000..5aa973c4 --- /dev/null +++ b/features/backup_restore.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +import argparse +import shutil + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--datadir", required=True) + parser.add_argument("--sourcedir", required=True) + args, _ = parser.parse_known_args() + + shutil.copytree(args.sourcedir, args.datadir) diff --git a/features/backup_restore.sh b/features/backup_restore.sh deleted file mode 100755 index 686a08e1..00000000 --- a/features/backup_restore.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -x - -while getopts ":-:" optchar; do - [[ "${optchar}" == "-" ]] || continue - case "${OPTARG}" in - datadir=* ) - PGDATA=${OPTARG#*=} - ;; - sourcedir=* ) - SOURCE=${OPTARG#*=} - ;; - esac -done - -[[ -z $PGDATA || -z $SOURCE ]] && exit 1 - -mkdir -p $(dirname $PGDATA) - -exec cp -af $SOURCE $PGDATA diff --git a/features/callback.py b/features/callback.py new file mode 100755 index 00000000..767be49f --- /dev/null +++ b/features/callback.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +import os +import psycopg2 +import sys + + +if __name__ == '__main__': + if not (len(sys.argv) >= 3 and sys.argv[3] == "master"): + sys.exit(1) + + os.environ['PGPASSWORD'] = 'zalando' + connection = psycopg2.connect(host='127.0.0.1', port=sys.argv[1], user='postgres') + cursor = connection.cursor() + cursor.execute("SELECT slot_name FROM pg_replication_slots WHERE slot_type = 'logical'") + + with open("data/postgres0/label", "w") as label: + label.write(next(iter(cursor.fetchone()), "")) diff --git a/features/callback.sh b/features/callback.sh deleted file mode 100755 index 27365a4e..00000000 --- a/features/callback.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -[[ "$3" == "master" ]] || exit - -PGPASSWORD=zalando psql -h localhost -U postgres -p $1 -w -tAc "SELECT slot_name FROM pg_replication_slots WHERE slot_type = 'logical'" >> data/postgres0/label diff --git a/features/callback2.py b/features/callback2.py new file mode 100755 index 00000000..6dad933a --- /dev/null +++ b/features/callback2.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python + +import sys +with open("data/{0}/{0}_cb.log".format(sys.argv[1]), "a+") as log: + log.write(" ".join(sys.argv[-3:]) + "\n") diff --git a/features/environment.py b/features/environment.py index 3ffca4df..d678a523 100644 --- a/features/environment.py +++ b/features/environment.py @@ -259,9 +259,9 @@ class PatroniController(AbstractController): return 'postgres://{username}:{password}@{host}:{port}/{database}'.format(**self._replication) def backup(self, dest=os.path.join('data', 'basebackup')): - subprocess.call([PatroniPoolController.BACKUP_SCRIPT, '--walmethod=none', - '--datadir=' + os.path.join(self._work_directory, dest), - '--dbname=' + self.backup_source]) + subprocess.call(PatroniPoolController.BACKUP_SCRIPT + ['--walmethod=none', + '--datadir=' + os.path.join(self._work_directory, dest), + '--dbname=' + self.backup_source]) class ProcessHang(object): @@ -532,7 +532,8 @@ class ExhibitorController(ZooKeeperController): class PatroniPoolController(object): - BACKUP_SCRIPT = 'features/backup_create.sh' + BACKUP_SCRIPT = [sys.executable, 'features/backup_create.py'] + ARCHIVE_RESTORE_SCRIPT = ' '.join((sys.executable, os.path.abspath('features/archive-restore.py'))) def __init__(self, context): self._context = context @@ -593,7 +594,7 @@ class PatroniPoolController(object): 'bootstrap': { 'method': 'pg_basebackup', 'pg_basebackup': { - 'command': self.BACKUP_SCRIPT + ' --walmethod=stream --dbname=' + f.backup_source + 'command': " ".join(self.BACKUP_SCRIPT) + ' --walmethod=stream --dbname=' + f.backup_source }, 'dcs': { 'postgresql': { @@ -606,8 +607,9 @@ class PatroniPoolController(object): 'postgresql': { 'parameters': { 'archive_mode': 'on', - 'archive_command': 'mkdir -p {0} && test ! -f {0}/%f && cp %p {0}/%f'.format( - os.path.join(self.patroni_path, 'data', 'wal_archive')) + 'archive_command': (self.ARCHIVE_RESTORE_SCRIPT + ' --mode archive ' + + '--dirname {} --filename %f --pathname %p').format( + os.path.join(self.patroni_path, 'data', 'wal_archive')) }, 'authentication': { 'superuser': {'password': 'zalando1'}, @@ -623,12 +625,14 @@ class PatroniPoolController(object): 'bootstrap': { 'method': 'backup_restore', 'backup_restore': { - 'command': 'features/backup_restore.sh --sourcedir=' + os.path.join(self.patroni_path, - 'data', 'basebackup'), + 'command': (sys.executable + ' features/backup_restore.py --sourcedir=' + + os.path.join(self.patroni_path, 'data', 'basebackup')), 'recovery_conf': { 'recovery_target_action': 'promote', 'recovery_target_timeline': 'latest', - 'restore_command': 'cp {0}/data/wal_archive/%f %p'.format(self.patroni_path) + 'restore_command': (self.ARCHIVE_RESTORE_SCRIPT + ' --mode restore ' + + '--dirname {} --filename %f --pathname %p').format( + os.path.join(self.patroni_path, 'data', 'wal_archive')) } } }, diff --git a/features/standby_cluster.feature b/features/standby_cluster.feature index 9ba3dd06..44310290 100644 --- a/features/standby_cluster.feature +++ b/features/standby_cluster.feature @@ -30,7 +30,7 @@ Feature: standby cluster When I issue a GET request to http://127.0.0.1:8009/standby_leader Then I receive a response code 200 And I receive a response role standby_leader - And there is a postgres1_cb.log with "on_start replica batman1\non_role_change standby_leader batman1" in postgres1 data directory + And there is a postgres1_cb.log with "on_role_change standby_leader batman1" in postgres1 data directory When I start postgres2 in a cluster batman1 Then postgres2 role is the replica after 24 seconds And table foo is present on postgres2 after 20 seconds diff --git a/features/steps/cascading_replication.py b/features/steps/cascading_replication.py index f8ca5708..a7610885 100644 --- a/features/steps/cascading_replication.py +++ b/features/steps/cascading_replication.py @@ -13,7 +13,7 @@ def start_patroni_with_a_name_value_tag(context, name, tag_name, tag_value): def check_label(context, label, content, name): label = context.pctl.read_label(name, label) label = label.replace('\n', '\\n') - assert label == content, "{0} is not equal to {1}".format(label, content) + assert content in label, "{0} doesn't contain {1}".format(label, content) @step('I create label with "{content:w}" in {name:w} data directory') diff --git a/features/steps/standby_cluster.py b/features/steps/standby_cluster.py index 03c62e5f..063d9f82 100644 --- a/features/steps/standby_cluster.py +++ b/features/steps/standby_cluster.py @@ -1,4 +1,5 @@ import os +import sys import time from behave import step @@ -9,7 +10,7 @@ SELECT * FROM pg_catalog.pg_stat_replication WHERE application_name = '{0}' """ -callback = "bash -c 'echo \"${*: -3:1} ${*: -2:1} ${*: -1:1}\" >> data/$1/$1_cb.log' -- " +callback = sys.executable + " features/callback2.py " @step('I start {name:w} with callback configured') @@ -17,7 +18,7 @@ def start_patroni_with_callbacks(context, name): return context.pctl.start(name, custom_config={ "postgresql": { "callbacks": { - "on_role_change": "features/callback.sh" + "on_role_change": sys.executable + " features/callback.py" } } }) @@ -30,8 +31,8 @@ def start_patroni(context, name, cluster_name): "postgresql": { "callbacks": {c: callback + name for c in ('on_start', 'on_stop', 'on_restart', 'on_role_change')}, "backup_restore": { - "command": "features/backup_restore.sh --sourcedir=" + os.path.join(context.pctl.patroni_path, - 'data', 'basebackup')} + "command": (sys.executable + " features/backup_restore.py --sourcedir=" + + os.path.join(context.pctl.patroni_path, 'data', 'basebackup'))} } })