diff --git a/helpers/postgresql.py b/helpers/postgresql.py index 2c527ad2..05279fdd 100644 --- a/helpers/postgresql.py +++ b/helpers/postgresql.py @@ -1,6 +1,8 @@ import logging import os import psycopg2 +import shutil +import subprocess import sys import time @@ -42,7 +44,13 @@ class Postgresql: self.superuser = config['superuser'] self.admin = config['admin'] self.recovery_conf = os.path.join(self.data_dir, 'recovery.conf') + self.configuration_to_save = (os.path.join(self.data_dir, 'pg_hba.conf'), + os.path.join(self.data_dir, 'postgresql.conf')) self._pg_ctl = 'pg_ctl -w -D ' + self.data_dir + self.wal_e = config.get('wal_e', None) + if self.wal_e: + self.wal_e_path = 'envdir {} wal-e --aws-instance-profile '.\ + format(self.wal_e.get('env_dir', '/home/postgres/etc/wal-e.d/env')) self.config = config @@ -101,11 +109,98 @@ class Postgresql: try: os.environ['PGPASSFILE'] = pgpass - return os.system('pg_basebackup -R -D {data_dir} --host={hostname} --port={port} -U {username}'.format( - data_dir=self.data_dir, **r)) == 0 + return self.create_replica(leader.address, r) == 0 finally: os.environ.pop('PGPASSFILE') + def create_replica(self, master_connurl, master_connection): + """ creates a new replica using either pg_basebackup or WAL-E """ + if self.should_use_s3_to_create_replica(master_connurl): + result = self.create_replica_with_s3() + # if restore from the backup on S3 failed - try with the pg_basebackup + if result == 0: + return result + return self.create_replica_with_pg_basebackup(master_connection) + + def create_replica_with_pg_basebackup(self, master_connection): + return os.system('pg_basebackup -R -D {data_dir} --host={hostname} --port={port} -U {username}'.format( + data_dir=self.data_dir, **master_connection)) + + def create_replica_with_s3(self): + if not self.wal_e or not self.wal_e_path: + return 1 + + ret = os.system(self.wal_e_path + ' backup-fetch {} LATEST'.format(self.data_dir)) + self.restore_configuration_files() + return ret + + def should_use_s3_to_create_replica(self, master_connurl): + """ determine whether it makes sense to use S3 and not pg_basebackup """ + if not self.wal_e or not self.wal_e_path: + return False + + threshold_megabytes = self.wal_e.get('threshold_megabytes', 10240) + threshold_backup_size_percentage = self.wal_e.get('threshold_backup_size_percentage', 30) + + try: + latest_backup = subprocess.check_output(self.wal_e_path.split() + ['backup-list', '--detail', 'LATEST']) + #name last_modified expanded_size_bytes wal_segment_backup_start wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop + # base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 20310671 00000001000000000000007F 00000040 00000001000000000000007F 00000240 + backup_strings = latest_backup.splitlines() if latest_backup else () + if len(backup_strings) != 2: + return False + + names = backup_strings[0].split() + vals = backup_strings[1].split() + if (len(names) != len(vals)) or (len(names) != 7): + return False + + backup_info = dict(zip(names, vals)) + except subprocess.CalledProcessError as e: + logger.error("could not query wal-e latest backup: {}".format(e)) + return False + + try: + backup_size = backup_info['expanded_size_bytes'] + backup_start_segment = backup_info['wal_segment_backup_start'] + backup_start_offset = backup_info['wal_segment_offset_backup_start'] + except Exception as e: + logger.error("unable to get some of S3 backup parameters: {}".format(e)) + return False + + # WAL filename is XXXXXXXXYYYYYYYY000000ZZ, where X - timeline, Y - LSN logical log file, + # ZZ - 2 high digits of LSN offset. The rest of the offset is the provided decimal offset, + # that we have to convert to hex and 'prepend' to the high offset digits. + + lsn_segment = backup_start_segment[8:16] + # first 2 characters of the result are 0x and the last one is L + lsn_offset = hex((long(backup_start_segment[16:32], 16) << 24) + long(backup_start_offset))[2:-1] + + # construct the LSN from the segment and offset + backup_start_lsn = '{}/{}'.format(lsn_segment, lsn_offset) + + conn = None + cursor = None + diff_in_bytes = long(backup_size) + try: + # get the difference in bytes between the current WAL location and the backup start offset + conn = psycopg2.connect(master_connurl) + conn.autocommit = True + cursor = conn.cursor() + cursor.execute("SELECT pg_xlog_location_diff(pg_current_xlog_location(), %s)", (backup_start_lsn,)) + diff_in_bytes = long(cursor.fetchone()[0]) + except psycopg2.Error as e: + logger.error('could not determine difference with the master location: {}'.format(e)) + return False + finally: + cursor and cursor.close() + conn and conn.close() + + # if the size of the accumulated WAL segments is more than a certan percentage of the backup size + # or exceeds the pre-determined size - pg_basebackup is chosen instead. + return (diff_in_bytes < long(threshold_megabytes) * 1048576) and\ + (diff_in_bytes < long(backup_size) * float(threshold_backup_size_percentage)/100) + def is_leader(self): return not self.query('SELECT pg_is_in_recovery()').fetchone()[0] @@ -125,6 +220,7 @@ class Postgresql: ret = os.system(self._pg_ctl + ' start -o "{}"'.format(self.server_options())) == 0 ret and self.load_replication_slots() + self.save_configuration_files() return ret def stop(self): @@ -219,6 +315,22 @@ primary_conninfo = '{}' self.write_recovery_conf(leader) self.restart() + def save_configuration_files(self): + """ + copy postgresql.conf to postgresql.conf.backup to preserve it in the WAL-e backup. + see http://comments.gmane.org/gmane.comp.db.postgresql.wal-e/239 + """ + for f in self.configuration_to_save: + shutil.copy(f, f+'.backup') + + def restore_configuration_files(self): + """ restore a previously saved postgresql.conf """ + try: + for f in self.configuration_to_save: + shutil.copy(f+'.backup', f) + except Exception as e: + logger.error("unable to restore configuration from WAL-E backup: {}".format(e)) + def promote(self): return os.system(self._pg_ctl + ' promote') == 0 diff --git a/postgres0.yml b/postgres0.yml index da630242..263cd693 100644 --- a/postgres0.yml +++ b/postgres0.yml @@ -19,6 +19,10 @@ postgresql: admin: username: admin password: admin + wal_e: + env_dir: /home/postgres/etc/wal-e.d/env + threshold_megabytes: 10240 + threshold_backup_size_percentage: 30 #recovery_conf: #restore_command: cp ../wal_archive/%f %p parameters: diff --git a/postgres1.yml b/postgres1.yml index e8f958b5..6d0082b2 100644 --- a/postgres1.yml +++ b/postgres1.yml @@ -21,6 +21,10 @@ postgresql: password: admin #recovery_conf: #restore_command: cp ../wal_archive/%f %p + wal_e: + env_dir: /home/postgres/etc/wal-e.d/env + threshold_megabytes: 10240 + threshold_backup_size_percentage: 30 parameters: archive_mode: "on" wal_level: hot_standby