From 169288c46c16d9941118728581317c5a941de628 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Mon, 18 May 2015 18:05:04 +0200
Subject: [PATCH 01/29] add functionality to fetch base backup from wal-e for
 the new replica creation and code to decide whether to use wal-e or
 pg_basebackup.

---
 helpers/postgresql.py | 77 +++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 75 insertions(+), 2 deletions(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index 185777e3..d8f31b1c 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -2,6 +2,7 @@ import logging
 import os
 import psycopg2
 import re
+import subprocess
 import sys
 import time
 
@@ -44,6 +45,7 @@ class Postgresql:
         self.admin = config['admin']
         self.recovery_conf = os.path.join(self.data_dir, 'recovery.conf')
         self._pg_ctl = 'pg_ctl -w -D ' + self.data_dir
+        self._wal_e = 'envdir {} wal-e --aws-instance-profile '.format(os.environ.get('WALE_ENV_DIR', '/home/postgres/etc/wal-e.d/env'))
 
         self.config = config
 
@@ -105,11 +107,82 @@ class Postgresql:
 
         try:
             os.environ['PGPASSFILE'] = pgpass
-            return os.system('pg_basebackup -R -D {data_dir} --host={hostname} --port={port} -U {username}'.format(
-                data_dir=self.data_dir, **r)) == 0
+            return self.create_replica(leader.address, r) == 0
         finally:
             os.environ.pop('PGPASSFILE')
 
+    def create_replica(self, master_connurl, master_connection):
+        """ creates a new replica using either pg_basebackup or WAL-E """
+        if self.should_use_s3_to_create_replica(master_connurl):
+            return self.create_replica_with_s3()
+        else:
+            return self.create_replica_with_pg_basebackup(master_connection)
+
+    def create_replica_with_pg_basebackup(self, master_connection):
+        return os.system('pg_basebackup -R -D {data_dir} --host={hostname} --port={port} -U {username}'.format(
+                data_dir=self.data_dir, **master_connection))
+
+    def create_replica_with_s3(self):
+        return os.system(self._wal_e + ' backup-fetch {} LATEST'.format(self.data_dir))
+
+    def should_use_s3_to_create_replica(self, master_connurl):
+        """ determine whether it makes sense to use S3 and not pg_basebackup """
+        try:
+            latest_backup = subprocess.check_output(self._wal_e + ' backup-list --detail LATEST')
+            #name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop
+            # base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z    20310671    00000001000000000000007F    00000040    00000001000000000000007F    00000240
+            backup_strings = latest_backup.splitlines() if latest_backup else ()
+            if len(backup_strings) != 2:
+                return False
+
+            names = backup_strings[0].split()
+            vals = backup_strings[1].split()
+            if (len(names) != len(vals)) or (len(names) != 7):
+                return False
+
+            backup_info = dict(zip(names, vals))
+        except subprocess.CalledProcessError as e:
+            logger.error("could not query wal-e latest backup: {}".format(e))
+            return False
+
+        try:
+            backup_size = backup_info['expanded_size_bytes']
+            backup_start_segment = backup_info['wal_segment_backup_start']
+            backup_start_offset = backup_info('wal_segment_offset_backup_start')
+        except Exception as e:
+            logger.error("unable to get some of S3 backup parameters: {}".format(e))
+            return False
+
+        # WAL filename is XXXXXXXXYYYYYYYY000000ZZ, where X - timeline, Y - LSN logical log file,
+        # ZZ - 2 high digits of LSN offset. The rest of the offset is the provided decimal offset,
+        # that we have to convert to hex and 'prepend' to the high offset digits.
+
+        lsn_segment = backup_start_segment[8:16]
+        lsn_offset = hex(int(backup_start_segment[16:32], 16) << 24 + backup_start_offset)[2:]
+
+        # construct the LSN from the segment and offset
+        backup_start_lsn = '{}/{}'.format(lsn_segment, lsn_offset)
+
+        conn = None
+        cursor = None
+        diff_in_bytes = backup_size
+        try:
+            # get the difference in bytes between the current WAL location and the backup start offset
+            conn = psycopg2.connect(master_connurl)
+            conn.autocommit = True
+            cursor = conn.cursor()
+            cursor.execute("SELECT pg_xlog_location_diff(pg_current_xlog_location(), %s)", (backup_start_lsn,))
+            diff_in_bytes = long(cursor.fetchone()[0])
+        except psycopg2.Error as e:
+            logger.error('could not determine difference with the master location: {}'.format(e))
+            return False
+        finally:
+            cursor and cursor.close()
+            conn and conn.close()
+
+        # if the size of the accumulated WAL segments is more than 5% of the backup size - we should use the pg_basebackup
+        return diff_in_bytes < long(backup_size) * 0.05
+
     def is_leader(self):
         return not self.query('SELECT pg_is_in_recovery()').fetchone()[0]
 

From 20fa4b21ca1f3b13873ce9429e7125354580eaa7 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 09:19:53 +0200
Subject: [PATCH 02/29] split arguments for check_output when checking for the
 latest backup.

---
 helpers/postgresql.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index d8f31b1c..6f9d4856 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -128,7 +128,7 @@ class Postgresql:
     def should_use_s3_to_create_replica(self, master_connurl):
         """ determine whether it makes sense to use S3 and not pg_basebackup """
         try:
-            latest_backup = subprocess.check_output(self._wal_e + ' backup-list --detail LATEST')
+            latest_backup = subprocess.check_output(self._wal_e.split() + ['backup-list', '--detail', 'LATEST'])
             #name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop
             # base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z    20310671    00000001000000000000007F    00000040    00000001000000000000007F    00000240
             backup_strings = latest_backup.splitlines() if latest_backup else ()

From ea5b4e9725ea1d54ab59756a32bfa40af811d213 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 10:07:12 +0200
Subject: [PATCH 03/29] fix a typo.

---
 helpers/postgresql.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index 6f9d4856..2242e4ec 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -148,7 +148,7 @@ class Postgresql:
         try:
             backup_size = backup_info['expanded_size_bytes']
             backup_start_segment = backup_info['wal_segment_backup_start']
-            backup_start_offset = backup_info('wal_segment_offset_backup_start')
+            backup_start_offset = backup_info['wal_segment_offset_backup_start']
         except Exception as e:
             logger.error("unable to get some of S3 backup parameters: {}".format(e))
             return False

From 9fc412a6737486ed84d0bf48c3d080952dbfe7af Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 10:32:48 +0200
Subject: [PATCH 04/29] some arithmetics and type conversion fixes.

---
 helpers/postgresql.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index 2242e4ec..a589ee4d 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -158,14 +158,15 @@ class Postgresql:
         # that we have to convert to hex and 'prepend' to the high offset digits.
 
         lsn_segment = backup_start_segment[8:16]
-        lsn_offset = hex(int(backup_start_segment[16:32], 16) << 24 + backup_start_offset)[2:]
+        # first 2 characters of the result are 0x and the last one is L
+        lsn_offset = hex((long(backup_start_segment[16:32], 16) << 24) + long(backup_start_offset))[2:-1]
 
         # construct the LSN from the segment and offset
         backup_start_lsn = '{}/{}'.format(lsn_segment, lsn_offset)
 
         conn = None
         cursor = None
-        diff_in_bytes = backup_size
+        diff_in_bytes = long(backup_size)
         try:
             # get the difference in bytes between the current WAL location and the backup start offset
             conn = psycopg2.connect(master_connurl)

From ae631aae698b3751b6e1beb49678e976695c5ffc Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 12:01:45 +0200
Subject: [PATCH 05/29] save/restore postgresql.conf for WAL-e backups, since
 WAL-E excludes postgresql.conf from the backup/restore.

---
 helpers/postgresql.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index a589ee4d..5b050799 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -2,6 +2,7 @@ import logging
 import os
 import psycopg2
 import re
+import shutil
 import subprocess
 import sys
 import time
@@ -44,6 +45,7 @@ class Postgresql:
         self.superuser = config['superuser']
         self.admin = config['admin']
         self.recovery_conf = os.path.join(self.data_dir, 'recovery.conf')
+        self.postgresql_conf = os.path.join(self.data_dir, 'postgresql.conf')
         self._pg_ctl = 'pg_ctl -w -D ' + self.data_dir
         self._wal_e = 'envdir {} wal-e --aws-instance-profile '.format(os.environ.get('WALE_ENV_DIR', '/home/postgres/etc/wal-e.d/env'))
 
@@ -91,6 +93,7 @@ class Postgresql:
 
     def initialize(self):
         if os.system(self._pg_ctl + ' initdb') == 0:
+            self.save_postgresql_conf()
             self.write_pg_hba()
 
             return True
@@ -123,7 +126,10 @@ class Postgresql:
                 data_dir=self.data_dir, **master_connection))
 
     def create_replica_with_s3(self):
-        return os.system(self._wal_e + ' backup-fetch {} LATEST'.format(self.data_dir))
+        ret = os.system(self._wal_e + ' backup-fetch {} LATEST'.format(self.data_dir))
+        self.restore_postgresql_conf()
+        return ret
+
 
     def should_use_s3_to_create_replica(self, master_connurl):
         """ determine whether it makes sense to use S3 and not pg_basebackup """
@@ -301,6 +307,20 @@ primary_conninfo = '{}'
         self.write_recovery_conf(leader)
         self.restart()
 
+    def save_posgresql_conf(self):
+        """
+            copy postgresql.conf to postgresql.conf.backup to preserve it in the WAL-e backup.
+            see http://comments.gmane.org/gmane.comp.db.postgresql.wal-e/239
+        """
+        shutil.copy(self.postgresql_conf, self.postgresql_conf+'.backup')
+
+    def restore_postgresql_conf(self):
+        """ restore a previously saved postgresql.conf """
+        try:
+            shutil.copy(self.postgresql_conf+'.backup', self.postgresql_conf)
+        except Exception as e:
+            logger.error("unable to restore postgresql.conf from WAL-E backup: {}".format(e))
+
     def promote(self):
         return os.system(self._pg_ctl + ' promote') == 0
 

From 380f0a27ac9ea3be9b1aa2f7c313887eaa3b7c0c Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 13:56:42 +0200
Subject: [PATCH 06/29] save postgresql.conf not only on initdb, but on any
 start of the server.

---
 helpers/postgresql.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index 5b050799..b6816f14 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -93,7 +93,6 @@ class Postgresql:
 
     def initialize(self):
         if os.system(self._pg_ctl + ' initdb') == 0:
-            self.save_postgresql_conf()
             self.write_pg_hba()
 
             return True
@@ -209,6 +208,7 @@ class Postgresql:
 
         ret = os.system(self._pg_ctl + ' start -o "{}"'.format(self.server_options())) == 0
         ret and self.load_replication_slots()
+        self.save_postgresql_conf()
         return ret
 
     def stop(self):

From 36282a10b25609fd4cef9577385e9a4a93b9e8ad Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 14:04:51 +0200
Subject: [PATCH 07/29] fix a typo.

---
 helpers/postgresql.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index b6816f14..eeda5ee7 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -307,7 +307,7 @@ primary_conninfo = '{}'
         self.write_recovery_conf(leader)
         self.restart()
 
-    def save_posgresql_conf(self):
+    def save_postgresql_conf(self):
         """
             copy postgresql.conf to postgresql.conf.backup to preserve it in the WAL-e backup.
             see http://comments.gmane.org/gmane.comp.db.postgresql.wal-e/239

From a8a5c104ceac0186808ceaf510cc30788fbaf239 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 14:24:01 +0200
Subject: [PATCH 08/29] write pg_hba.conf for the replica initialized from
 WAL-E. Temporarily use WAL-e always as a mean to create replica for testing
 purposes.

---
 helpers/postgresql.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index eeda5ee7..a2528ca6 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -127,6 +127,7 @@ class Postgresql:
     def create_replica_with_s3(self):
         ret = os.system(self._wal_e + ' backup-fetch {} LATEST'.format(self.data_dir))
         self.restore_postgresql_conf()
+        self.write_pg_hba()
         return ret
 
 
@@ -187,7 +188,7 @@ class Postgresql:
             conn and conn.close()
 
         # if the size of the accumulated WAL segments is more than 5% of the backup size - we should use the pg_basebackup
-        return diff_in_bytes < long(backup_size) * 0.05
+        return True or (diff_in_bytes < long(backup_size) * 0.05)
 
     def is_leader(self):
         return not self.query('SELECT pg_is_in_recovery()').fetchone()[0]

From d8d9da12ffb3cea645010c1df4d1f709b2f5c976 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 14:33:13 +0200
Subject: [PATCH 09/29] save/restore both postgresql.conf and pg_hba.conf for
 WAL-e backup.

---
 helpers/postgresql.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index a2528ca6..0a837640 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -45,7 +45,7 @@ class Postgresql:
         self.superuser = config['superuser']
         self.admin = config['admin']
         self.recovery_conf = os.path.join(self.data_dir, 'recovery.conf')
-        self.postgresql_conf = os.path.join(self.data_dir, 'postgresql.conf')
+        self.configuration_to_save = (os.path.join(self.data_dir, 'pg_hba.conf'), os.path.join(self.data_dir, 'postgresql.conf'))
         self._pg_ctl = 'pg_ctl -w -D ' + self.data_dir
         self._wal_e = 'envdir {} wal-e --aws-instance-profile '.format(os.environ.get('WALE_ENV_DIR', '/home/postgres/etc/wal-e.d/env'))
 
@@ -126,8 +126,7 @@ class Postgresql:
 
     def create_replica_with_s3(self):
         ret = os.system(self._wal_e + ' backup-fetch {} LATEST'.format(self.data_dir))
-        self.restore_postgresql_conf()
-        self.write_pg_hba()
+        self.restore_configuration_files()
         return ret
 
 
@@ -209,7 +208,7 @@ class Postgresql:
 
         ret = os.system(self._pg_ctl + ' start -o "{}"'.format(self.server_options())) == 0
         ret and self.load_replication_slots()
-        self.save_postgresql_conf()
+        self.save_configuration_files()
         return ret
 
     def stop(self):
@@ -308,19 +307,21 @@ primary_conninfo = '{}'
         self.write_recovery_conf(leader)
         self.restart()
 
-    def save_postgresql_conf(self):
+    def save_configuration_files(self):
         """
             copy postgresql.conf to postgresql.conf.backup to preserve it in the WAL-e backup.
             see http://comments.gmane.org/gmane.comp.db.postgresql.wal-e/239
         """
-        shutil.copy(self.postgresql_conf, self.postgresql_conf+'.backup')
+        for f in self.configuration_to_save:
+            shutil.copy(f, f+'.backup')
 
-    def restore_postgresql_conf(self):
+    def restore_configuration_files(self):
         """ restore a previously saved postgresql.conf """
         try:
-            shutil.copy(self.postgresql_conf+'.backup', self.postgresql_conf)
+            for f in self.configuration_to_save:
+                shutil.copy(f+'.backup', f)
         except Exception as e:
-            logger.error("unable to restore postgresql.conf from WAL-E backup: {}".format(e))
+            logger.error("unable to restore configuration from WAL-E backup: {}".format(e))
 
     def promote(self):
         return os.system(self._pg_ctl + ' promote') == 0

From 9e25b3a78c13820afe121c3501a1710a82fcfb37 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 14:41:20 +0200
Subject: [PATCH 10/29] returned the choice between S3 and pg_basebackup based
 on diff_in_bytes

---
 helpers/postgresql.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index 0a837640..b9b74763 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -187,7 +187,7 @@ class Postgresql:
             conn and conn.close()
 
         # if the size of the accumulated WAL segments is more than 5% of the backup size - we should use the pg_basebackup
-        return True or (diff_in_bytes < long(backup_size) * 0.05)
+        return diff_in_bytes < long(backup_size) * 0.05
 
     def is_leader(self):
         return not self.query('SELECT pg_is_in_recovery()').fetchone()[0]

From ab17836aee729e66ee5b7a06878d6d508bc04303 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 14:45:35 +0200
Subject: [PATCH 11/29] revert the S3/pg_basebackup choice to unconditionally
 prefer S3 for testing purposes.

---
 helpers/postgresql.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index b9b74763..0a837640 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -187,7 +187,7 @@ class Postgresql:
             conn and conn.close()
 
         # if the size of the accumulated WAL segments is more than 5% of the backup size - we should use the pg_basebackup
-        return diff_in_bytes < long(backup_size) * 0.05
+        return True or (diff_in_bytes < long(backup_size) * 0.05)
 
     def is_leader(self):
         return not self.query('SELECT pg_is_in_recovery()').fetchone()[0]

From 2191b0227d397e2fa38d25688b7b399e13b33b49 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 14:56:36 +0200
Subject: [PATCH 12/29] Put back the S3/pg_basebackup condition after testing.

---
 helpers/postgresql.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index 0a837640..b9b74763 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -187,7 +187,7 @@ class Postgresql:
             conn and conn.close()
 
         # if the size of the accumulated WAL segments is more than 5% of the backup size - we should use the pg_basebackup
-        return True or (diff_in_bytes < long(backup_size) * 0.05)
+        return diff_in_bytes < long(backup_size) * 0.05
 
     def is_leader(self):
         return not self.query('SELECT pg_is_in_recovery()').fetchone()[0]

From 6a082139487aa24699924cdd696d183c95c01326 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 17:16:07 +0200
Subject: [PATCH 13/29] retry with pg_basebackup if one cannot get data from
 WAL-E.

---
 helpers/postgresql.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index b9b74763..c9743d74 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -116,9 +116,11 @@ class Postgresql:
     def create_replica(self, master_connurl, master_connection):
         """ creates a new replica using either pg_basebackup or WAL-E """
         if self.should_use_s3_to_create_replica(master_connurl):
-            return self.create_replica_with_s3()
-        else:
-            return self.create_replica_with_pg_basebackup(master_connection)
+            result = self.create_replica_with_s3()
+            # if restore from the backup on S3 failed - try with the pg_basebackup
+            if result == 0:
+                return result
+        return self.create_replica_with_pg_basebackup(master_connection)
 
     def create_replica_with_pg_basebackup(self, master_connection):
         return os.system('pg_basebackup -R -D {data_dir} --host={hostname} --port={port} -U {username}'.format(

From 5a579b0c9229e9e0886729093307f16885b257ba Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Wed, 20 May 2015 10:20:52 +0200
Subject: [PATCH 14/29] use governor yaml parameters and not environment
 variables to configure backups with WAL-E.

---
 helpers/postgresql.py | 28 +++++++++++++++++++++-------
 postgres0.yml         |  4 ++++
 postgres1.yml         |  4 ++++
 3 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index c9743d74..4fbc8b49 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -45,9 +45,13 @@ class Postgresql:
         self.superuser = config['superuser']
         self.admin = config['admin']
         self.recovery_conf = os.path.join(self.data_dir, 'recovery.conf')
-        self.configuration_to_save = (os.path.join(self.data_dir, 'pg_hba.conf'), os.path.join(self.data_dir, 'postgresql.conf'))
+        self.configuration_to_save = (os.path.join(self.data_dir, 'pg_hba.conf'),
+                                      os.path.join(self.data_dir, 'postgresql.conf'))
         self._pg_ctl = 'pg_ctl -w -D ' + self.data_dir
-        self._wal_e = 'envdir {} wal-e --aws-instance-profile '.format(os.environ.get('WALE_ENV_DIR', '/home/postgres/etc/wal-e.d/env'))
+        self.wal_e = config.get('wal_e', None)
+        if self.wal_e:
+            self.wal_e_path = 'envdir {} wal-e --aws-instance-profile '.\
+                               format(self.wal_e.get('env_dir', '/home/postgres/etc/wal-e.d/env'))
 
         self.config = config
 
@@ -127,15 +131,23 @@ class Postgresql:
                 data_dir=self.data_dir, **master_connection))
 
     def create_replica_with_s3(self):
-        ret = os.system(self._wal_e + ' backup-fetch {} LATEST'.format(self.data_dir))
+        if not self.wal_e or not self.wal_e_path:
+            return 1
+
+        ret = os.system(self.wal_e_path + ' backup-fetch {} LATEST'.format(self.data_dir))
         self.restore_configuration_files()
         return ret
 
-
     def should_use_s3_to_create_replica(self, master_connurl):
         """ determine whether it makes sense to use S3 and not pg_basebackup """
+        if not self.wal_e or not self.wal_e_path:
+            return False
+
+        threshold_megabytes = self.wal_e.get('threshold_megabytes', 10240)
+        threshold_backup_size_percentage = self.wal_e.get('threshold_backup_size_percentage', 30)
+
         try:
-            latest_backup = subprocess.check_output(self._wal_e.split() + ['backup-list', '--detail', 'LATEST'])
+            latest_backup = subprocess.check_output(self.wal_e_path.split() + ['backup-list', '--detail', 'LATEST'])
             #name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop
             # base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z    20310671    00000001000000000000007F    00000040    00000001000000000000007F    00000240
             backup_strings = latest_backup.splitlines() if latest_backup else ()
@@ -188,8 +200,10 @@ class Postgresql:
             cursor and cursor.close()
             conn and conn.close()
 
-        # if the size of the accumulated WAL segments is more than 5% of the backup size - we should use the pg_basebackup
-        return diff_in_bytes < long(backup_size) * 0.05
+        # if the size of the accumulated WAL segments is more than a certan percentage of the backup size
+        # or exceeds the pre-determined size - pg_basebackup is chosen instead.
+        return (diff_in_bytes < long(threshold_megabytes) * 1048576) and\
+               (diff_in_bytes < long(backup_size) * float(threshold_backup_size_percentage)/100)
 
     def is_leader(self):
         return not self.query('SELECT pg_is_in_recovery()').fetchone()[0]
diff --git a/postgres0.yml b/postgres0.yml
index da630242..263cd693 100644
--- a/postgres0.yml
+++ b/postgres0.yml
@@ -19,6 +19,10 @@ postgresql:
   admin:
     username: admin
     password: admin
+  wal_e:
+    env_dir: /home/postgres/etc/wal-e.d/env
+    threshold_megabytes: 10240
+    threshold_backup_size_percentage: 30
   #recovery_conf:
     #restore_command: cp ../wal_archive/%f %p
   parameters:
diff --git a/postgres1.yml b/postgres1.yml
index e8f958b5..6d0082b2 100644
--- a/postgres1.yml
+++ b/postgres1.yml
@@ -21,6 +21,10 @@ postgresql:
     password: admin
   #recovery_conf:
     #restore_command: cp ../wal_archive/%f %p
+  wal_e:
+    env_dir: /home/postgres/etc/wal-e.d/env
+    threshold_megabytes: 10240
+    threshold_backup_size_percentage: 30
   parameters:
     archive_mode: "on"
     wal_level: hot_standby

From 9e82c3ebd7e7c80d5a156ff0fd5e97b291c58995 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Mon, 18 May 2015 18:05:04 +0200
Subject: [PATCH 15/29] add functionality to fetch base backup from wal-e for
 the new replica creation and code to decide whether to use wal-e or
 pg_basebackup.

---
 helpers/postgresql.py | 77 +++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 75 insertions(+), 2 deletions(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index 14bfbeb7..74af6c7e 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -1,6 +1,7 @@
 import logging
 import os
 import psycopg2
+import subprocess
 import sys
 import time
 
@@ -43,6 +44,7 @@ class Postgresql:
         self.admin = config['admin']
         self.recovery_conf = os.path.join(self.data_dir, 'recovery.conf')
         self._pg_ctl = 'pg_ctl -w -D ' + self.data_dir
+        self._wal_e = 'envdir {} wal-e --aws-instance-profile '.format(os.environ.get('WALE_ENV_DIR', '/home/postgres/etc/wal-e.d/env'))
 
         self.config = config
 
@@ -104,11 +106,82 @@ class Postgresql:
 
         try:
             os.environ['PGPASSFILE'] = pgpass
-            return os.system('pg_basebackup -R -D {data_dir} --host={hostname} --port={port} -U {username}'.format(
-                data_dir=self.data_dir, **r)) == 0
+            return self.create_replica(leader.address, r) == 0
         finally:
             os.environ.pop('PGPASSFILE')
 
+    def create_replica(self, master_connurl, master_connection):
+        """ creates a new replica using either pg_basebackup or WAL-E """
+        if self.should_use_s3_to_create_replica(master_connurl):
+            return self.create_replica_with_s3()
+        else:
+            return self.create_replica_with_pg_basebackup(master_connection)
+
+    def create_replica_with_pg_basebackup(self, master_connection):
+        return os.system('pg_basebackup -R -D {data_dir} --host={hostname} --port={port} -U {username}'.format(
+                data_dir=self.data_dir, **master_connection))
+
+    def create_replica_with_s3(self):
+        return os.system(self._wal_e + ' backup-fetch {} LATEST'.format(self.data_dir))
+
+    def should_use_s3_to_create_replica(self, master_connurl):
+        """ determine whether it makes sense to use S3 and not pg_basebackup """
+        try:
+            latest_backup = subprocess.check_output(self._wal_e + ' backup-list --detail LATEST')
+            #name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop
+            # base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z    20310671    00000001000000000000007F    00000040    00000001000000000000007F    00000240
+            backup_strings = latest_backup.splitlines() if latest_backup else ()
+            if len(backup_strings) != 2:
+                return False
+
+            names = backup_strings[0].split()
+            vals = backup_strings[1].split()
+            if (len(names) != len(vals)) or (len(names) != 7):
+                return False
+
+            backup_info = dict(zip(names, vals))
+        except subprocess.CalledProcessError as e:
+            logger.error("could not query wal-e latest backup: {}".format(e))
+            return False
+
+        try:
+            backup_size = backup_info['expanded_size_bytes']
+            backup_start_segment = backup_info['wal_segment_backup_start']
+            backup_start_offset = backup_info('wal_segment_offset_backup_start')
+        except Exception as e:
+            logger.error("unable to get some of S3 backup parameters: {}".format(e))
+            return False
+
+        # WAL filename is XXXXXXXXYYYYYYYY000000ZZ, where X - timeline, Y - LSN logical log file,
+        # ZZ - 2 high digits of LSN offset. The rest of the offset is the provided decimal offset,
+        # that we have to convert to hex and 'prepend' to the high offset digits.
+
+        lsn_segment = backup_start_segment[8:16]
+        lsn_offset = hex(int(backup_start_segment[16:32], 16) << 24 + backup_start_offset)[2:]
+
+        # construct the LSN from the segment and offset
+        backup_start_lsn = '{}/{}'.format(lsn_segment, lsn_offset)
+
+        conn = None
+        cursor = None
+        diff_in_bytes = backup_size
+        try:
+            # get the difference in bytes between the current WAL location and the backup start offset
+            conn = psycopg2.connect(master_connurl)
+            conn.autocommit = True
+            cursor = conn.cursor()
+            cursor.execute("SELECT pg_xlog_location_diff(pg_current_xlog_location(), %s)", (backup_start_lsn,))
+            diff_in_bytes = long(cursor.fetchone()[0])
+        except psycopg2.Error as e:
+            logger.error('could not determine difference with the master location: {}'.format(e))
+            return False
+        finally:
+            cursor and cursor.close()
+            conn and conn.close()
+
+        # if the size of the accumulated WAL segments is more than 5% of the backup size - we should use the pg_basebackup
+        return diff_in_bytes < long(backup_size) * 0.05
+
     def is_leader(self):
         return not self.query('SELECT pg_is_in_recovery()').fetchone()[0]
 

From 19a04a81dab9a8b8b67374a2564fa3096f444903 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 09:19:53 +0200
Subject: [PATCH 16/29] split arguments for check_output when checking for the
 latest backup.

---
 helpers/postgresql.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index 74af6c7e..7c8f049f 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -127,7 +127,7 @@ class Postgresql:
     def should_use_s3_to_create_replica(self, master_connurl):
         """ determine whether it makes sense to use S3 and not pg_basebackup """
         try:
-            latest_backup = subprocess.check_output(self._wal_e + ' backup-list --detail LATEST')
+            latest_backup = subprocess.check_output(self._wal_e.split() + ['backup-list', '--detail', 'LATEST'])
             #name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop
             # base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z    20310671    00000001000000000000007F    00000040    00000001000000000000007F    00000240
             backup_strings = latest_backup.splitlines() if latest_backup else ()

From 594efe3b64f8274ed52f7a0eba73782e7fef7749 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 10:07:12 +0200
Subject: [PATCH 17/29] fix a typo.

---
 helpers/postgresql.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index 7c8f049f..c66b4344 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -147,7 +147,7 @@ class Postgresql:
         try:
             backup_size = backup_info['expanded_size_bytes']
             backup_start_segment = backup_info['wal_segment_backup_start']
-            backup_start_offset = backup_info('wal_segment_offset_backup_start')
+            backup_start_offset = backup_info['wal_segment_offset_backup_start']
         except Exception as e:
             logger.error("unable to get some of S3 backup parameters: {}".format(e))
             return False

From d4b366a7e2665bc8beb7335320c84fe29fea71e3 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 10:32:48 +0200
Subject: [PATCH 18/29] some arithmetics and type conversion fixes.

---
 helpers/postgresql.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index c66b4344..5d89b579 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -157,14 +157,15 @@ class Postgresql:
         # that we have to convert to hex and 'prepend' to the high offset digits.
 
         lsn_segment = backup_start_segment[8:16]
-        lsn_offset = hex(int(backup_start_segment[16:32], 16) << 24 + backup_start_offset)[2:]
+        # first 2 characters of the result are 0x and the last one is L
+        lsn_offset = hex((long(backup_start_segment[16:32], 16) << 24) + long(backup_start_offset))[2:-1]
 
         # construct the LSN from the segment and offset
         backup_start_lsn = '{}/{}'.format(lsn_segment, lsn_offset)
 
         conn = None
         cursor = None
-        diff_in_bytes = backup_size
+        diff_in_bytes = long(backup_size)
         try:
             # get the difference in bytes between the current WAL location and the backup start offset
             conn = psycopg2.connect(master_connurl)

From 7a2be131bd675769bfdb2e19731d58b434c892e7 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 12:01:45 +0200
Subject: [PATCH 19/29] save/restore postgresql.conf for WAL-e backups, since
 WAL-E excludes postgresql.conf from the backup/restore.

---
 helpers/postgresql.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index 5d89b579..7bf16270 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -1,6 +1,7 @@
 import logging
 import os
 import psycopg2
+import shutil
 import subprocess
 import sys
 import time
@@ -43,6 +44,7 @@ class Postgresql:
         self.superuser = config['superuser']
         self.admin = config['admin']
         self.recovery_conf = os.path.join(self.data_dir, 'recovery.conf')
+        self.postgresql_conf = os.path.join(self.data_dir, 'postgresql.conf')
         self._pg_ctl = 'pg_ctl -w -D ' + self.data_dir
         self._wal_e = 'envdir {} wal-e --aws-instance-profile '.format(os.environ.get('WALE_ENV_DIR', '/home/postgres/etc/wal-e.d/env'))
 
@@ -89,7 +91,8 @@ class Postgresql:
         return not os.path.exists(self.data_dir) or os.listdir(self.data_dir) == []
 
     def initialize(self):
-        if os.system(self._pg_ctl + ' initdb -o --encoding=UTF8') == 0:
+        if os.system(self._pg_ctl + ' initdb') == 0:
+            self.save_postgresql_conf()
             self.write_pg_hba()
 
             return True
@@ -122,7 +125,10 @@ class Postgresql:
                 data_dir=self.data_dir, **master_connection))
 
     def create_replica_with_s3(self):
-        return os.system(self._wal_e + ' backup-fetch {} LATEST'.format(self.data_dir))
+        ret = os.system(self._wal_e + ' backup-fetch {} LATEST'.format(self.data_dir))
+        self.restore_postgresql_conf()
+        return ret
+
 
     def should_use_s3_to_create_replica(self, master_connurl):
         """ determine whether it makes sense to use S3 and not pg_basebackup """
@@ -296,6 +302,20 @@ primary_conninfo = '{}'
         self.write_recovery_conf(leader)
         self.restart()
 
+    def save_posgresql_conf(self):
+        """
+            copy postgresql.conf to postgresql.conf.backup to preserve it in the WAL-e backup.
+            see http://comments.gmane.org/gmane.comp.db.postgresql.wal-e/239
+        """
+        shutil.copy(self.postgresql_conf, self.postgresql_conf+'.backup')
+
+    def restore_postgresql_conf(self):
+        """ restore a previously saved postgresql.conf """
+        try:
+            shutil.copy(self.postgresql_conf+'.backup', self.postgresql_conf)
+        except Exception as e:
+            logger.error("unable to restore postgresql.conf from WAL-E backup: {}".format(e))
+
     def promote(self):
         return os.system(self._pg_ctl + ' promote') == 0
 

From 75b30d80149cc3afaf5cbfd7094f570bc80e3f65 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 13:56:42 +0200
Subject: [PATCH 20/29] save postgresql.conf not only on initdb, but on any
 start of the server.

---
 helpers/postgresql.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index 7bf16270..2d17dd26 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -92,7 +92,6 @@ class Postgresql:
 
     def initialize(self):
         if os.system(self._pg_ctl + ' initdb') == 0:
-            self.save_postgresql_conf()
             self.write_pg_hba()
 
             return True
@@ -208,6 +207,7 @@ class Postgresql:
 
         ret = os.system(self._pg_ctl + ' start -o "{}"'.format(self.server_options())) == 0
         ret and self.load_replication_slots()
+        self.save_postgresql_conf()
         return ret
 
     def stop(self):

From 8a6a7b4cea08230f99113f22e41e26b461d58bb6 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 14:04:51 +0200
Subject: [PATCH 21/29] fix a typo.

---
 helpers/postgresql.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index 2d17dd26..b31d1d0f 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -302,7 +302,7 @@ primary_conninfo = '{}'
         self.write_recovery_conf(leader)
         self.restart()
 
-    def save_posgresql_conf(self):
+    def save_postgresql_conf(self):
         """
             copy postgresql.conf to postgresql.conf.backup to preserve it in the WAL-e backup.
             see http://comments.gmane.org/gmane.comp.db.postgresql.wal-e/239

From 42dfaaab3f8a94642cbe6b61a0ee733f136dfe8b Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 14:24:01 +0200
Subject: [PATCH 22/29] write pg_hba.conf for the replica initialized from
 WAL-E. Temporarily use WAL-e always as a mean to create replica for testing
 purposes.

---
 helpers/postgresql.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index b31d1d0f..660de55d 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -126,6 +126,7 @@ class Postgresql:
     def create_replica_with_s3(self):
         ret = os.system(self._wal_e + ' backup-fetch {} LATEST'.format(self.data_dir))
         self.restore_postgresql_conf()
+        self.write_pg_hba()
         return ret
 
 
@@ -186,7 +187,7 @@ class Postgresql:
             conn and conn.close()
 
         # if the size of the accumulated WAL segments is more than 5% of the backup size - we should use the pg_basebackup
-        return diff_in_bytes < long(backup_size) * 0.05
+        return True or (diff_in_bytes < long(backup_size) * 0.05)
 
     def is_leader(self):
         return not self.query('SELECT pg_is_in_recovery()').fetchone()[0]

From e02eec91bf631550aea1f463e677388d57f3faf1 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 14:33:13 +0200
Subject: [PATCH 23/29] save/restore both postgresql.conf and pg_hba.conf for
 WAL-e backup.

---
 helpers/postgresql.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index 660de55d..216c61dd 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -44,7 +44,7 @@ class Postgresql:
         self.superuser = config['superuser']
         self.admin = config['admin']
         self.recovery_conf = os.path.join(self.data_dir, 'recovery.conf')
-        self.postgresql_conf = os.path.join(self.data_dir, 'postgresql.conf')
+        self.configuration_to_save = (os.path.join(self.data_dir, 'pg_hba.conf'), os.path.join(self.data_dir, 'postgresql.conf'))
         self._pg_ctl = 'pg_ctl -w -D ' + self.data_dir
         self._wal_e = 'envdir {} wal-e --aws-instance-profile '.format(os.environ.get('WALE_ENV_DIR', '/home/postgres/etc/wal-e.d/env'))
 
@@ -125,8 +125,7 @@ class Postgresql:
 
     def create_replica_with_s3(self):
         ret = os.system(self._wal_e + ' backup-fetch {} LATEST'.format(self.data_dir))
-        self.restore_postgresql_conf()
-        self.write_pg_hba()
+        self.restore_configuration_files()
         return ret
 
 
@@ -208,7 +207,7 @@ class Postgresql:
 
         ret = os.system(self._pg_ctl + ' start -o "{}"'.format(self.server_options())) == 0
         ret and self.load_replication_slots()
-        self.save_postgresql_conf()
+        self.save_configuration_files()
         return ret
 
     def stop(self):
@@ -303,19 +302,21 @@ primary_conninfo = '{}'
         self.write_recovery_conf(leader)
         self.restart()
 
-    def save_postgresql_conf(self):
+    def save_configuration_files(self):
         """
             copy postgresql.conf to postgresql.conf.backup to preserve it in the WAL-e backup.
             see http://comments.gmane.org/gmane.comp.db.postgresql.wal-e/239
         """
-        shutil.copy(self.postgresql_conf, self.postgresql_conf+'.backup')
+        for f in self.configuration_to_save:
+            shutil.copy(f, f+'.backup')
 
-    def restore_postgresql_conf(self):
+    def restore_configuration_files(self):
         """ restore a previously saved postgresql.conf """
         try:
-            shutil.copy(self.postgresql_conf+'.backup', self.postgresql_conf)
+            for f in self.configuration_to_save:
+                shutil.copy(f+'.backup', f)
         except Exception as e:
-            logger.error("unable to restore postgresql.conf from WAL-E backup: {}".format(e))
+            logger.error("unable to restore configuration from WAL-E backup: {}".format(e))
 
     def promote(self):
         return os.system(self._pg_ctl + ' promote') == 0

From 2efa97334b2ecd9ac2220d5df9d520897ab00740 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 14:41:20 +0200
Subject: [PATCH 24/29] returned the choice between S3 and pg_basebackup based
 on diff_in_bytes

---
 helpers/postgresql.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index 216c61dd..eacd17e3 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -186,7 +186,7 @@ class Postgresql:
             conn and conn.close()
 
         # if the size of the accumulated WAL segments is more than 5% of the backup size - we should use the pg_basebackup
-        return True or (diff_in_bytes < long(backup_size) * 0.05)
+        return diff_in_bytes < long(backup_size) * 0.05
 
     def is_leader(self):
         return not self.query('SELECT pg_is_in_recovery()').fetchone()[0]

From 4af752d6686e885c52bbeb947021e683308a3a7d Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 14:45:35 +0200
Subject: [PATCH 25/29] revert the S3/pg_basebackup choice to unconditionally
 prefer S3 for testing purposes.

---
 helpers/postgresql.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index eacd17e3..216c61dd 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -186,7 +186,7 @@ class Postgresql:
             conn and conn.close()
 
         # if the size of the accumulated WAL segments is more than 5% of the backup size - we should use the pg_basebackup
-        return diff_in_bytes < long(backup_size) * 0.05
+        return True or (diff_in_bytes < long(backup_size) * 0.05)
 
     def is_leader(self):
         return not self.query('SELECT pg_is_in_recovery()').fetchone()[0]

From 3898610ea347063c0e26c0ec043459bf162b7d78 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 14:56:36 +0200
Subject: [PATCH 26/29] Put back the S3/pg_basebackup condition after testing.

---
 helpers/postgresql.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index 216c61dd..eacd17e3 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -186,7 +186,7 @@ class Postgresql:
             conn and conn.close()
 
         # if the size of the accumulated WAL segments is more than 5% of the backup size - we should use the pg_basebackup
-        return True or (diff_in_bytes < long(backup_size) * 0.05)
+        return diff_in_bytes < long(backup_size) * 0.05
 
     def is_leader(self):
         return not self.query('SELECT pg_is_in_recovery()').fetchone()[0]

From 8a36f17d21edea7ae9ec6a4d8aced614c5400e57 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 19 May 2015 17:16:07 +0200
Subject: [PATCH 27/29] retry with pg_basebackup if one cannot get data from
 WAL-E.

---
 helpers/postgresql.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index eacd17e3..f923c5ac 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -115,9 +115,11 @@ class Postgresql:
     def create_replica(self, master_connurl, master_connection):
         """ creates a new replica using either pg_basebackup or WAL-E """
         if self.should_use_s3_to_create_replica(master_connurl):
-            return self.create_replica_with_s3()
-        else:
-            return self.create_replica_with_pg_basebackup(master_connection)
+            result = self.create_replica_with_s3()
+            # if restore from the backup on S3 failed - try with the pg_basebackup
+            if result == 0:
+                return result
+        return self.create_replica_with_pg_basebackup(master_connection)
 
     def create_replica_with_pg_basebackup(self, master_connection):
         return os.system('pg_basebackup -R -D {data_dir} --host={hostname} --port={port} -U {username}'.format(

From 1c1a15908c9ced562e3825edcef87b6d5e8a5940 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Wed, 20 May 2015 10:20:52 +0200
Subject: [PATCH 28/29] use governor yaml parameters and not environment
 variables to configure backups with WAL-E.

---
 helpers/postgresql.py | 28 +++++++++++++++++++++-------
 postgres0.yml         |  4 ++++
 postgres1.yml         |  4 ++++
 3 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index f923c5ac..6bcac842 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -44,9 +44,13 @@ class Postgresql:
         self.superuser = config['superuser']
         self.admin = config['admin']
         self.recovery_conf = os.path.join(self.data_dir, 'recovery.conf')
-        self.configuration_to_save = (os.path.join(self.data_dir, 'pg_hba.conf'), os.path.join(self.data_dir, 'postgresql.conf'))
+        self.configuration_to_save = (os.path.join(self.data_dir, 'pg_hba.conf'),
+                                      os.path.join(self.data_dir, 'postgresql.conf'))
         self._pg_ctl = 'pg_ctl -w -D ' + self.data_dir
-        self._wal_e = 'envdir {} wal-e --aws-instance-profile '.format(os.environ.get('WALE_ENV_DIR', '/home/postgres/etc/wal-e.d/env'))
+        self.wal_e = config.get('wal_e', None)
+        if self.wal_e:
+            self.wal_e_path = 'envdir {} wal-e --aws-instance-profile '.\
+                               format(self.wal_e.get('env_dir', '/home/postgres/etc/wal-e.d/env'))
 
         self.config = config
 
@@ -126,15 +130,23 @@ class Postgresql:
                 data_dir=self.data_dir, **master_connection))
 
     def create_replica_with_s3(self):
-        ret = os.system(self._wal_e + ' backup-fetch {} LATEST'.format(self.data_dir))
+        if not self.wal_e or not self.wal_e_path:
+            return 1
+
+        ret = os.system(self.wal_e_path + ' backup-fetch {} LATEST'.format(self.data_dir))
         self.restore_configuration_files()
         return ret
 
-
     def should_use_s3_to_create_replica(self, master_connurl):
         """ determine whether it makes sense to use S3 and not pg_basebackup """
+        if not self.wal_e or not self.wal_e_path:
+            return False
+
+        threshold_megabytes = self.wal_e.get('threshold_megabytes', 10240)
+        threshold_backup_size_percentage = self.wal_e.get('threshold_backup_size_percentage', 30)
+
         try:
-            latest_backup = subprocess.check_output(self._wal_e.split() + ['backup-list', '--detail', 'LATEST'])
+            latest_backup = subprocess.check_output(self.wal_e_path.split() + ['backup-list', '--detail', 'LATEST'])
             #name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop
             # base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z    20310671    00000001000000000000007F    00000040    00000001000000000000007F    00000240
             backup_strings = latest_backup.splitlines() if latest_backup else ()
@@ -187,8 +199,10 @@ class Postgresql:
             cursor and cursor.close()
             conn and conn.close()
 
-        # if the size of the accumulated WAL segments is more than 5% of the backup size - we should use the pg_basebackup
-        return diff_in_bytes < long(backup_size) * 0.05
+        # if the size of the accumulated WAL segments is more than a certan percentage of the backup size
+        # or exceeds the pre-determined size - pg_basebackup is chosen instead.
+        return (diff_in_bytes < long(threshold_megabytes) * 1048576) and\
+               (diff_in_bytes < long(backup_size) * float(threshold_backup_size_percentage)/100)
 
     def is_leader(self):
         return not self.query('SELECT pg_is_in_recovery()').fetchone()[0]
diff --git a/postgres0.yml b/postgres0.yml
index da630242..263cd693 100644
--- a/postgres0.yml
+++ b/postgres0.yml
@@ -19,6 +19,10 @@ postgresql:
   admin:
     username: admin
     password: admin
+  wal_e:
+    env_dir: /home/postgres/etc/wal-e.d/env
+    threshold_megabytes: 10240
+    threshold_backup_size_percentage: 30
   #recovery_conf:
     #restore_command: cp ../wal_archive/%f %p
   parameters:
diff --git a/postgres1.yml b/postgres1.yml
index e8f958b5..6d0082b2 100644
--- a/postgres1.yml
+++ b/postgres1.yml
@@ -21,6 +21,10 @@ postgresql:
     password: admin
   #recovery_conf:
     #restore_command: cp ../wal_archive/%f %p
+  wal_e:
+    env_dir: /home/postgres/etc/wal-e.d/env
+    threshold_megabytes: 10240
+    threshold_backup_size_percentage: 30
   parameters:
     archive_mode: "on"
     wal_level: hot_standby

From 7dd1e295cad8b729a29a8ae8a144afd19ceeb94c Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Wed, 20 May 2015 16:31:26 +0200
Subject: [PATCH 29/29] re-add the encoding parameter to initdb.

---
 helpers/postgresql.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index 6bcac842..0035628f 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -95,7 +95,7 @@ class Postgresql:
         return not os.path.exists(self.data_dir) or os.listdir(self.data_dir) == []
 
     def initialize(self):
-        if os.system(self._pg_ctl + ' initdb') == 0:
+        if os.system(self._pg_ctl + ' initdb --encoding=UTF8') == 0:
             self.write_pg_hba()
 
             return True