mirror of
https://github.com/outbackdingo/patroni.git
synced 2026-09-01 09:09:21 +00:00
Oleksii Kliukin
b629e0852f
If patroni detects the former master was killed, it runs it first in a single-user mode and then shuts down normally, to make sure pg_rewind will see a normal shut down status in pg_controldata. Add a flag need_rewind, since the point where it is detected that rewind might be necessary is moved out the code that runs rewind.
393 lines
16 KiB
Python
393 lines
16 KiB
Python
import mock # for the mock.call method, importing it without a namespace breaks python3
|
|
import os
|
|
import psycopg2
|
|
import shutil
|
|
import unittest
|
|
|
|
from sys import version_info
|
|
if version_info.major == 2:
|
|
import __builtin__ as builtins
|
|
else:
|
|
import builtins
|
|
|
|
from mock import Mock, MagicMock, patch
|
|
from patroni.dcs import Cluster, Leader, Member
|
|
from patroni.exceptions import PostgresException, PostgresConnectionException
|
|
from patroni.postgresql import Postgresql
|
|
from patroni.utils import RetryFailedError
|
|
from test_ha import false
|
|
import subprocess
|
|
|
|
|
|
class MockCursor:
|
|
|
|
def __init__(self, connection):
|
|
self.connection = connection
|
|
self.closed = False
|
|
self.results = []
|
|
|
|
def execute(self, sql, *params):
|
|
if sql.startswith('blabla') or sql == 'CHECKPOINT':
|
|
raise psycopg2.OperationalError()
|
|
elif sql.startswith('RetryFailedError'):
|
|
raise RetryFailedError('retry')
|
|
elif sql.startswith('SELECT slot_name'):
|
|
self.results = [('blabla',), ('foobar',)]
|
|
elif sql.startswith('SELECT pg_current_xlog_location()'):
|
|
self.results = [(0,)]
|
|
elif sql.startswith('SELECT pg_is_in_recovery(), %s'):
|
|
if params[0][0] == 1:
|
|
raise psycopg2.OperationalError()
|
|
elif params[0][0] == 2:
|
|
self.results = [(True, -1)]
|
|
else:
|
|
self.results = [(False, 0)]
|
|
elif sql.startswith('SELECT pg_xlog_location_diff'):
|
|
self.results = [(0,)]
|
|
elif sql.startswith('SELECT pg_is_in_recovery()'):
|
|
self.results = [(False, )]
|
|
elif sql.startswith('SELECT to_char(pg_postmaster_start_time'):
|
|
self.results = [('', True, '', '', '', False)]
|
|
else:
|
|
self.results = [(
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
)]
|
|
|
|
def fetchone(self):
|
|
return self.results[0]
|
|
|
|
def close(self):
|
|
pass
|
|
|
|
def __iter__(self):
|
|
for i in self.results:
|
|
yield i
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
pass
|
|
|
|
|
|
class MockConnect(Mock):
|
|
|
|
autocommit = False
|
|
closed = 0
|
|
|
|
def cursor(self):
|
|
return MockCursor(self)
|
|
|
|
|
|
def pg_controldata_string(*args, **kwargs):
|
|
return """
|
|
pg_control version number: 942
|
|
Catalog version number: 201509161
|
|
Database system identifier: 6200971513092291716
|
|
Database cluster state: shut down in recovery
|
|
pg_control last modified: Fri Oct 2 10:57:06 2015
|
|
Latest checkpoint location: 0/30000C8
|
|
Prior checkpoint location: 0/2000060
|
|
Latest checkpoint's REDO location: 0/3000090
|
|
Latest checkpoint's REDO WAL file: 000000020000000000000003
|
|
Latest checkpoint's TimeLineID: 2
|
|
Latest checkpoint's PrevTimeLineID: 2
|
|
Latest checkpoint's full_page_writes: on
|
|
Latest checkpoint's NextXID: 0/943
|
|
Latest checkpoint's NextOID: 24576
|
|
Latest checkpoint's NextMultiXactId: 1
|
|
Latest checkpoint's NextMultiOffset: 0
|
|
Latest checkpoint's oldestXID: 931
|
|
Latest checkpoint's oldestXID's DB: 1
|
|
Latest checkpoint's oldestActiveXID: 943
|
|
Latest checkpoint's oldestMultiXid: 1
|
|
Latest checkpoint's oldestMulti's DB: 1
|
|
Latest checkpoint's oldestCommitTs: 0
|
|
Latest checkpoint's newestCommitTs: 0
|
|
Time of latest checkpoint: Fri Oct 2 10:56:54 2015
|
|
Fake LSN counter for unlogged rels: 0/1
|
|
Minimum recovery ending location: 0/30241F8
|
|
Min recovery ending loc's timeline: 2
|
|
Backup start location: 0/0
|
|
Backup end location: 0/0
|
|
End-of-backup record required: no
|
|
wal_level setting: hot_standby
|
|
wal_log_hints setting: on
|
|
max_connections setting: 100
|
|
max_worker_processes setting: 8
|
|
max_prepared_xacts setting: 0
|
|
max_locks_per_xact setting: 64
|
|
track_commit_timestamp setting: off
|
|
Maximum data alignment: 8
|
|
Database block size: 8192
|
|
Blocks per segment of large relation: 131072
|
|
WAL block size: 8192
|
|
Bytes per WAL segment: 16777216
|
|
Maximum length of identifiers: 64
|
|
Maximum columns in an index: 32
|
|
Maximum size of a TOAST chunk: 1996
|
|
Size of a large-object chunk: 2048
|
|
Date/time type storage: 64-bit integers
|
|
Float4 argument passing: by value
|
|
Float8 argument passing: by value
|
|
Data page checksum version: 0
|
|
"""
|
|
|
|
|
|
def postmaster_opts_string(*args, **kwargs):
|
|
return '/usr/local/pgsql/bin/postgres "-D" "data/postgresql0" "--listen_addresses=127.0.0.1" "--port=5432"'\
|
|
' "--hot_standby=on" "--wal_keep_segments=8" "--wal_level=hot_standby" "--archive_command=mkdir -p ../wal_archive \n'\
|
|
'&& cp %p ../wal_archive/%f" "--wal_log_hints=on" "--max_wal_senders=5" "--archive_timeout=1800s" "--archive_mode=on"'\
|
|
' "--max_replication_slots=5"\n'
|
|
|
|
|
|
def psycopg2_connect(*args, **kwargs):
|
|
return MockConnect()
|
|
|
|
|
|
@patch('subprocess.call', Mock(return_value=0))
|
|
@patch('shutil.copy', Mock())
|
|
@patch('psycopg2.connect', psycopg2_connect)
|
|
class TestPostgresql(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': 'data/test0',
|
|
'listen': '127.0.0.1, *:5432', 'connect_address': '127.0.0.2:5432',
|
|
'pg_hba': ['hostssl all all 0.0.0.0/0 md5', 'host all all 0.0.0.0/0 md5'],
|
|
'superuser': {'password': ''},
|
|
'admin': {'username': 'admin', 'password': 'admin'},
|
|
'pg_rewind': {'username': 'admin', 'password': 'admin'},
|
|
'replication': {'username': 'replicator',
|
|
'password': 'rep-pass',
|
|
'network': '127.0.0.1/32'},
|
|
'parameters': {'foo': 'bar'}, 'recovery_conf': {'foo': 'bar'},
|
|
'callbacks': {'on_start': 'true', 'on_stop': 'true',
|
|
'on_restart': 'true', 'on_role_change': 'true',
|
|
'on_reload': 'true'
|
|
},
|
|
'restore': 'true'})
|
|
if not os.path.exists(self.p.data_dir):
|
|
os.makedirs(self.p.data_dir)
|
|
self.leadermem = Member(0, 'leader', 'postgres://replicator:[email protected]:5435/postgres', None, None, 28)
|
|
self.leader = Leader(-1, None, 28, self.leadermem)
|
|
self.other = Member(0, 'test1', 'postgres://replicator:[email protected]:5433/postgres', None, None, 28)
|
|
self.me = Member(0, 'test0', 'postgres://replicator:[email protected]:5434/postgres', None, None, 28)
|
|
|
|
def tearDown(self):
|
|
shutil.rmtree('data')
|
|
|
|
def test_data_directory_empty(self):
|
|
self.assertTrue(self.p.data_directory_empty())
|
|
|
|
def test_initialize(self):
|
|
self.assertTrue(self.p.initialize())
|
|
self.assertTrue(os.path.exists(os.path.join(self.p.data_dir, 'pg_hba.conf')))
|
|
|
|
def test_start_stop(self):
|
|
self.assertFalse(self.p.start())
|
|
self.p.is_running = false
|
|
with open(os.path.join(self.p.data_dir, 'postmaster.pid'), 'w'):
|
|
pass
|
|
self.assertTrue(self.p.start())
|
|
self.assertTrue(self.p.stop())
|
|
|
|
def test_sync_from_leader(self):
|
|
self.assertTrue(self.p.sync_from_leader(self.leader))
|
|
|
|
@patch('subprocess.call', side_effect=Exception("Test"))
|
|
def test_pg_rewind(self, mock_call):
|
|
self.assertTrue(self.p.rewind(self.leader))
|
|
self.p
|
|
subprocess.call = mock_call
|
|
self.assertFalse(self.p.rewind(self.leader))
|
|
|
|
@patch('patroni.postgresql.Postgresql.rewind', return_value=False)
|
|
@patch('patroni.postgresql.Postgresql.remove_data_directory', MagicMock(return_value=True))
|
|
def test_follow_the_leader(self, mock_pg_rewind):
|
|
self.p.demote(self.leader)
|
|
self.p.follow_the_leader(None)
|
|
self.p.demote(self.leader)
|
|
self.p.follow_the_leader(self.leader)
|
|
self.p.follow_the_leader(Leader(-1, None, 28, self.other))
|
|
self.p.rewind = mock_pg_rewind
|
|
self.p.follow_the_leader(self.leader)
|
|
|
|
def test_create_replica(self):
|
|
self.p.delete_trigger_file = Mock(side_effect=OSError())
|
|
self.assertEquals(self.p.create_replica({'host': '', 'port': '', 'user': ''}, ''), 1)
|
|
|
|
def test_create_connection_users(self):
|
|
cfg = self.p.config
|
|
cfg['superuser']['username'] = 'test'
|
|
p = Postgresql(cfg)
|
|
p.create_connection_users()
|
|
|
|
def test_sync_replication_slots(self):
|
|
self.p.start()
|
|
cluster = Cluster(True, self.leader, 0, [self.me, self.other, self.leadermem])
|
|
self.p.sync_replication_slots(cluster)
|
|
|
|
@patch.object(MockConnect, 'closed', 2)
|
|
def test__query(self):
|
|
self.assertRaises(PostgresConnectionException, self.p._query, 'blabla')
|
|
|
|
def test_query(self):
|
|
self.p.query('select 1')
|
|
self.assertRaises(PostgresConnectionException, self.p.query, 'RetryFailedError')
|
|
self.assertRaises(psycopg2.OperationalError, self.p.query, 'blabla')
|
|
|
|
def test_is_healthiest_node(self):
|
|
cluster = Cluster(True, self.leader, 0, [self.me, self.other, self.leadermem])
|
|
self.assertTrue(self.p.is_healthiest_node(cluster))
|
|
self.p.is_leader = false
|
|
self.assertFalse(self.p.is_healthiest_node(cluster))
|
|
self.p.xlog_position = lambda: 1
|
|
self.assertTrue(self.p.is_healthiest_node(cluster))
|
|
self.p.xlog_position = lambda: 2
|
|
self.assertFalse(self.p.is_healthiest_node(cluster))
|
|
self.p.config['maximum_lag_on_failover'] = -3
|
|
self.assertFalse(self.p.is_healthiest_node(cluster))
|
|
|
|
def test_reload(self):
|
|
self.assertTrue(self.p.reload())
|
|
|
|
def test_is_healthy(self):
|
|
self.assertTrue(self.p.is_healthy())
|
|
self.p.is_running = false
|
|
self.assertFalse(self.p.is_healthy())
|
|
|
|
def test_promote(self):
|
|
self.assertTrue(self.p.promote())
|
|
self.assertTrue(self.p.promote())
|
|
|
|
def test_last_operation(self):
|
|
self.assertEquals(self.p.last_operation(), '0')
|
|
|
|
@patch('subprocess.Popen', Mock(side_effect=OSError()))
|
|
def test_call_nowait(self):
|
|
self.assertFalse(self.p.call_nowait('on_start'))
|
|
|
|
def test_non_existing_callback(self):
|
|
self.assertFalse(self.p.call_nowait('foobar'))
|
|
|
|
def test_is_leader_exception(self):
|
|
self.p.start()
|
|
self.p.query = Mock(side_effect=psycopg2.OperationalError("not supported"))
|
|
self.assertTrue(self.p.stop())
|
|
|
|
@patch('os.rename', Mock())
|
|
@patch('os.path.isdir', Mock(return_value=True))
|
|
def test_move_data_directory(self):
|
|
self.p.is_running = false
|
|
self.p.move_data_directory()
|
|
with patch('os.rename', Mock(side_effect=OSError())):
|
|
self.p.move_data_directory()
|
|
|
|
def test_bootstrap(self):
|
|
self.assertRaises(PostgresException, self.p.bootstrap)
|
|
self.p.start = Mock(return_value=True)
|
|
self.p.bootstrap()
|
|
|
|
def test_remove_data_directory(self):
|
|
self.p.data_dir = 'data_dir'
|
|
self.p.remove_data_directory()
|
|
os.mkdir(self.p.data_dir)
|
|
self.p.remove_data_directory()
|
|
open(self.p.data_dir, 'w').close()
|
|
self.p.remove_data_directory()
|
|
os.symlink('unexisting', self.p.data_dir)
|
|
with patch('os.unlink', Mock(side_effect=Exception)):
|
|
self.p.remove_data_directory()
|
|
self.p.remove_data_directory()
|
|
|
|
@patch('subprocess.check_output', MagicMock(return_value=0, side_effect=pg_controldata_string))
|
|
@patch('subprocess.check_output', side_effect=subprocess.CalledProcessError)
|
|
@patch('subprocess.check_output', side_effect=Exception('Failed'))
|
|
def test_controldata(self, check_output_call_error, check_output_generic_exception):
|
|
data = self.p.controldata()
|
|
self.assertEquals(len(data), 50)
|
|
self.assertEquals(data['Database cluster state'], 'shut down in recovery')
|
|
self.assertEquals(data['wal_log_hints setting'], 'on')
|
|
self.assertEquals(int(data['Database block size']), 8192)
|
|
|
|
subprocess.check_output = check_output_call_error
|
|
data = self.p.controldata()
|
|
self.assertIsNone(data)
|
|
|
|
subprocess.check_output = check_output_generic_exception
|
|
self.assertRaises(Exception, self.p.controldata())
|
|
|
|
def test_read_postmaster_opts(self):
|
|
m = mock.mock_open(read_data=postmaster_opts_string())
|
|
with patch.object(builtins, 'open', m):
|
|
data = self.p.read_postmaster_opts()
|
|
self.assertEquals(data['wal_level'], 'hot_standby')
|
|
self.assertEquals(int(data['max_replication_slots']), 5)
|
|
self.assertEqual(data.get('D'), None)
|
|
|
|
m.side_effect = IOError("foo")
|
|
data = self.p.read_postmaster_opts()
|
|
self.assertEqual(data, dict())
|
|
|
|
m.side_effect = Exception("foo")
|
|
self.assertRaises(Exception, self.p.read_postmaster_opts())
|
|
|
|
@patch('subprocess.Popen')
|
|
@patch.object(builtins, 'open', MagicMock(return_value=42))
|
|
def test_single_user_mode(self, subprocess_popen_mock):
|
|
subprocess_popen_mock.return_value.wait.return_value = 0
|
|
self.assertEquals(self.p.single_user_mode(options=dict(archive_mode='on', archive_command='false')), 0)
|
|
subprocess_popen_mock.assert_called_once_with(['postgres', '--single', '-D', self.p.data_dir,
|
|
'-c', 'archive_command=false', '-c', 'archive_mode=on',
|
|
'postgres'], stdin=subprocess.PIPE,
|
|
stdout=42,
|
|
stderr=subprocess.STDOUT)
|
|
subprocess_popen_mock.reset_mock()
|
|
self.assertEquals(self.p.single_user_mode(command="CHECKPOINT"), 0)
|
|
subprocess_popen_mock.assert_called_once_with(['postgres', '--single', '-D', self.p.data_dir,
|
|
'postgres'], stdin=subprocess.PIPE,
|
|
stdout=42,
|
|
stderr=subprocess.STDOUT)
|
|
subprocess_popen_mock.return_value = None
|
|
self.assertEquals(self.p.single_user_mode(), 1)
|
|
|
|
def fake_listdir(path):
|
|
if path.endswith(os.path.join('pg_xlog', 'archive_status')):
|
|
return ["a", "b", "c"]
|
|
return []
|
|
|
|
@patch('os.listdir', MagicMock(side_effect=fake_listdir))
|
|
@patch('os.path.isdir', MagicMock(return_value=True))
|
|
@patch('os.unlink', return_value=True)
|
|
@patch('os.remove', return_value=True)
|
|
@patch('os.path.islink', return_value=False)
|
|
@patch('os.path.isfile', return_value=True)
|
|
def test_cleanup_archive_status(self, mock_file, mock_link, mock_remove, mock_unlink):
|
|
ap = os.path.join(self.p.data_dir, 'pg_xlog', 'archive_status/')
|
|
self.p.cleanup_archive_status()
|
|
mock_remove.assert_has_calls([mock.call(ap+'a'), mock.call(ap+'b'), mock.call(ap+'c')])
|
|
mock_unlink.assert_not_called()
|
|
|
|
mock_remove.reset_mock()
|
|
mock_file.return_value = False
|
|
mock_link.return_value = True
|
|
self.p.cleanup_archive_status()
|
|
mock_unlink.assert_has_calls([mock.call(ap+'a'), mock.call(ap+'b'), mock.call(ap+'c')])
|
|
mock_remove.assert_not_called()
|
|
|
|
mock_unlink.reset_mock()
|
|
mock_remove.reset_mock()
|
|
mock_file.side_effect = Exception("foo")
|
|
self.p.cleanup_archive_status()
|
|
mock_unlink.assert_not_called()
|
|
mock_remove.assert_not_called()
|