From ebf64828e1eaf0e36aa303a9a99b738064909da5 Mon Sep 17 00:00:00 2001 From: Feike Steenbergen Date: Fri, 2 Sep 2016 16:37:06 +0200 Subject: [PATCH 1/3] Decode output from wal-e list backup When running this script using Python3, the output is bytestring instead of string. We explicitly decode it to ensure checks further down are ok. The trigger for this patch is: ERROR: unable to get some of WALE backup parameters: 'expanded_size_bytes' --- patroni/scripts/wale_restore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patroni/scripts/wale_restore.py b/patroni/scripts/wale_restore.py index 4383e986..28eefe2e 100755 --- a/patroni/scripts/wale_restore.py +++ b/patroni/scripts/wale_restore.py @@ -73,7 +73,7 @@ class WALERestore(object): # base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z # 20310671 00000001000000000000007F 00000040 # 00000001000000000000007F 00000240 - backup_strings = latest_backup.splitlines() if latest_backup else () + backup_strings = latest_backup.decode('utf-8').splitlines() if latest_backup else () if len(backup_strings) != 2: return False From 5ba1294d604b1df58f3c12db551a51684670b58d Mon Sep 17 00:00:00 2001 From: Feike Steenbergen Date: Fri, 2 Sep 2016 16:52:51 +0200 Subject: [PATCH 2/3] Fix tests for wal-e restore --- tests/test_wale_restore.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_wale_restore.py b/tests/test_wale_restore.py index 9e15dc9b..6f93be63 100644 --- a/tests/test_wale_restore.py +++ b/tests/test_wale_restore.py @@ -10,26 +10,25 @@ def fake_backup_data(self, *args, **kwargs): """ return the fake result of WAL-E backup-list""" return """name last_modified expanded_size_bytes wal_segment_backup_start wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 00000001000000000000007F 00000040 00000001000000000000007F 00000240 -""" - +""".encode('utf-8') def fake_backup_data_2(self, *args, **kwargs): """ return the fake result of WAL-E backup-list""" - return """name last_modified expanded_size_bytes wal_segment_backup_start wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop """ + return """name last_modified expanded_size_bytes wal_segment_backup_start wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop """.encode('utf-8') def fake_backup_data_3(self, *args, **kwargs): """ return the fake result of WAL-E backup-list""" return """name last_modified expanded_size_bytes wal_segment_backup_start wal_segment_offset_backup_start wal_segment_backup_stop base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 00000001000000000000007F 00000040 00000001000000000000007F 00000240 -""" +""".encode('utf-8') def fake_backup_data_4(self, *args, **kwargs): """ return the fake result of WAL-E backup-list""" return """name last_modified expanded_size_foo wal_segment_backup_start wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 00000001000000000000007F 00000040 00000001000000000000007F 00000240 -""" +""".encode('utf-8') @patch('os.access', MagicMock(return_value=True)) From 57a0ac90861e85ae6c1ff0667539dac289e3492e Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 5 Sep 2016 12:14:37 +0200 Subject: [PATCH 3/3] pep8 format of test_wale_restore.py --- tests/test_wale_restore.py | 37 ++++++++++--------------------------- 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/tests/test_wale_restore.py b/tests/test_wale_restore.py index 6f93be63..4e7080e0 100644 --- a/tests/test_wale_restore.py +++ b/tests/test_wale_restore.py @@ -6,29 +6,10 @@ from mock import MagicMock, patch, PropertyMock from patroni.scripts.wale_restore import WALERestore, main as _main -def fake_backup_data(self, *args, **kwargs): - """ return the fake result of WAL-E backup-list""" - return """name last_modified expanded_size_bytes wal_segment_backup_start wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop -base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 00000001000000000000007F 00000040 00000001000000000000007F 00000240 -""".encode('utf-8') - -def fake_backup_data_2(self, *args, **kwargs): - """ return the fake result of WAL-E backup-list""" - return """name last_modified expanded_size_bytes wal_segment_backup_start wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop """.encode('utf-8') - - -def fake_backup_data_3(self, *args, **kwargs): - """ return the fake result of WAL-E backup-list""" - return """name last_modified expanded_size_bytes wal_segment_backup_start wal_segment_offset_backup_start wal_segment_backup_stop -base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 00000001000000000000007F 00000040 00000001000000000000007F 00000240 -""".encode('utf-8') - - -def fake_backup_data_4(self, *args, **kwargs): - """ return the fake result of WAL-E backup-list""" - return """name last_modified expanded_size_foo wal_segment_backup_start wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop -base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 00000001000000000000007F 00000040 00000001000000000000007F 00000240 -""".encode('utf-8') +wale_output = b'name last_modified expanded_size_bytes wal_segment_backup_start ' +\ + b'wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop\n' +\ + b'base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 ' +\ + b'00000001000000000000007F 00000040 00000001000000000000007F 00000240\n' @patch('os.access', MagicMock(return_value=True)) @@ -38,7 +19,7 @@ base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 000 @patch('psycopg2.extensions.cursor', MagicMock(autospec=True)) @patch('psycopg2.extensions.connection', MagicMock(autospec=True)) @patch('psycopg2.connect', MagicMock(autospec=True)) -@patch('subprocess.check_output', MagicMock(side_effect=fake_backup_data)) +@patch('subprocess.check_output', MagicMock(return_value=wale_output)) class TestWALERestore(unittest.TestCase): def setUp(self): @@ -49,11 +30,13 @@ class TestWALERestore(unittest.TestCase): self.assertFalse(self.wale_restore.should_use_s3_to_create_replica()) with patch('subprocess.check_output', MagicMock(side_effect=subprocess.CalledProcessError(1, "cmd", "foo"))): self.assertFalse(self.wale_restore.should_use_s3_to_create_replica()) - with patch('subprocess.check_output', MagicMock(side_effect=fake_backup_data_2)): + with patch('subprocess.check_output', MagicMock(return_value=wale_output.split(b'\n')[0])): self.assertFalse(self.wale_restore.should_use_s3_to_create_replica()) - with patch('subprocess.check_output', MagicMock(side_effect=fake_backup_data_3)): + with patch('subprocess.check_output', + MagicMock(return_value=wale_output.replace(b' wal_segment_offset_backup_stop', b''))): self.assertFalse(self.wale_restore.should_use_s3_to_create_replica()) - with patch('subprocess.check_output', MagicMock(side_effect=fake_backup_data_4)): + with patch('subprocess.check_output', + MagicMock(return_value=wale_output.replace(b'expanded_size_bytes', b'expanded_size_foo'))): self.assertFalse(self.wale_restore.should_use_s3_to_create_replica()) self.wale_restore.should_use_s3_to_create_replica()