gitops/patroni
1
0
Fork 0
mirror of https://github.com/outbackdingo/patroni.git synced 2026-08-29 17:09:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:zalando/patroni into feature/citus-secondaries

Browse Source
This commit is contained in:
Alexander Kukushkin
2023-08-17 13:16:01 +02:00
parent a6e05b240c a4ac4963d1
commit e54b88534c
61 changed files with 3953 additions and 1149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/__init__.py
+24
View File
@@ -118,6 +118,21 @@ class MockCursor(object):
'"state":"streaming","sync_state":"async","sync_priority":0}]'
now = datetime.datetime.now(tzutc)
self.results = [(now, 0, '', 0, '', False, now, 'streaming', None, replication_info)]
elif sql.startswith('SELECT name, current_setting(name) FROM pg_settings'):
self.results = [('data_directory', 'data'),
('hba_file', os.path.join('data', 'pg_hba.conf')),
('ident_file', os.path.join('data', 'pg_ident.conf')),
('max_connections', 42),
('max_locks_per_transaction', 73),
('max_prepared_transactions', 0),
('max_replication_slots', 21),
('max_wal_senders', 37),
('track_commit_timestamp', 'off'),
('wal_level', 'replica'),
('listen_addresses', '6.6.6.6'),
('port', 1984),
('archive_command', 'my archive command'),
('cluster_name', 'my_cluster')]
elif sql.startswith('SELECT name, setting'):
self.results = [('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
('wal_block_size', '8192', None, 'integer', 'internal'),
@@ -163,11 +178,20 @@ class MockCursor(object):
pass
class MockConnectionInfo(object):
def parameter_status(self, param_name):
if param_name == 'is_superuser':
return 'on'
return '0'
class MockConnect(object):
server_version = 99999
autocommit = False
closed = 0
info = MockConnectionInfo()
def cursor(self):
return MockCursor(self)
tests/test_api.py
+1
View File
@@ -36,6 +36,7 @@ class MockPostgresql(object):
pending_restart = True
wal_name = 'wal'
lsn_name = 'lsn'
wal_flush = '_flush'
POSTMASTER_START_TIME = 'pg_catalog.pg_postmaster_start_time()'
TL_LSN = 'CASE WHEN pg_catalog.pg_is_in_recovery()'
citus_handler = Mock()
tests/test_config.py
+1
View File
@@ -85,6 +85,7 @@ class TestConfig(unittest.TestCase):
@patch('os.path.exists', Mock(return_value=True))
@patch('os.remove', Mock(side_effect=IOError))
@patch('os.close', Mock(side_effect=IOError))
@patch('os.chmod', Mock())
@patch('shutil.move', Mock(return_value=None))
@patch('json.dump', Mock())
def test_save_cache(self):
tests/test_config_generator.py
+334
View File
@@ -0,0 +1,334 @@
import os
import psutil
import socket
import unittest
from . import MockConnect, MockCursor, MockConnectionInfo
from copy import deepcopy
from mock import MagicMock, Mock, PropertyMock, mock_open, patch
from patroni.__main__ import main as _main
from patroni.config import Config
from patroni.config_generator import AbstractConfigGenerator, get_address
from patroni.utils import patch_config
from . import psycopg_connect
@patch('patroni.psycopg.connect', psycopg_connect)
@patch('socket.getaddrinfo', Mock(return_value=[(0, 0, 0, 0, ('1.9.8.4', 1984))]))
@patch('builtins.open', MagicMock())
@patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 16.2"))
@patch('psutil.Process.exe', Mock(return_value='/bin/dir/from/running/postgres'))
@patch('psutil.Process.__init__', Mock(return_value=None))
class TestGenerateConfig(unittest.TestCase):
no_value_msg = '#FIXME'
_HOSTNAME = socket.gethostname()
_IP = sorted(socket.getaddrinfo(_HOSTNAME, 0, socket.AF_UNSPEC, socket.SOCK_STREAM, 0), key=lambda x: x[0])[0][4][0]
def setUp(self):
self.maxDiff = None
os.environ['PATRONI_SCOPE'] = 'scope_from_env'
os.environ['PATRONI_POSTGRESQL_BIN_DIR'] = '/bin/from/env'
os.environ['PATRONI_SUPERUSER_USERNAME'] = 'su_user_from_env'
os.environ['PATRONI_SUPERUSER_PASSWORD'] = 'su_pwd_from_env'
os.environ['PATRONI_REPLICATION_USERNAME'] = 'repl_user_from_env'
os.environ['PATRONI_REPLICATION_PASSWORD'] = 'repl_pwd_from_env'
os.environ['PATRONI_REWIND_USERNAME'] = 'rewind_user_from_env'
os.environ['PGUSER'] = 'pguser_from_env'
os.environ['PGPASSWORD'] = 'pguser_pwd_from_env'
os.environ['PATRONI_RESTAPI_CONNECT_ADDRESS'] = 'localhost:8080'
os.environ['PATRONI_RESTAPI_LISTEN'] = 'localhost:8080'
os.environ['PATRONI_POSTGRESQL_BIN_POSTGRES'] = 'custom_postgres_bin_from_env'
self.environ = deepcopy(os.environ)
dynamic_config = Config.get_default_config()
dynamic_config['postgresql']['parameters'] = dict(dynamic_config['postgresql']['parameters'])
del dynamic_config['standby_cluster']
dynamic_config['postgresql']['parameters']['wal_keep_segments'] = 8
dynamic_config['postgresql']['use_pg_rewind'] = True
self.config = {
'scope': self.environ['PATRONI_SCOPE'],
'name': self._HOSTNAME,
'bootstrap': {
'dcs': dynamic_config
},
'postgresql': {
'connect_address': self.no_value_msg + ':5432',
'data_dir': self.no_value_msg,
'listen': self.no_value_msg + ':5432',
'pg_hba': ['host all all all md5',
f'host replication {self.environ["PATRONI_REPLICATION_USERNAME"]} all md5'],
'authentication': {'superuser': {'username': self.environ['PATRONI_SUPERUSER_USERNAME'],
'password': self.environ['PATRONI_SUPERUSER_PASSWORD']},
'replication': {'username': self.environ['PATRONI_REPLICATION_USERNAME'],
'password': self.environ['PATRONI_REPLICATION_PASSWORD']},
'rewind': {'username': self.environ['PATRONI_REWIND_USERNAME']}},
'bin_dir': self.environ['PATRONI_POSTGRESQL_BIN_DIR'],
'bin_name': {'postgres': self.environ['PATRONI_POSTGRESQL_BIN_POSTGRES']},
'parameters': {'password_encryption': 'md5'}
},
'restapi': {
'connect_address': self.environ['PATRONI_RESTAPI_CONNECT_ADDRESS'],
'listen': self.environ['PATRONI_RESTAPI_LISTEN']
}
}
def _set_running_instance_config_vals(self):
# values are taken from tests/__init__.py
conf = {
'scope': 'my_cluster',
'bootstrap': {
'dcs': {
'postgresql': {
'parameters': {
'max_connections': 42,
'max_locks_per_transaction': 73,
'max_replication_slots': 21,
'max_wal_senders': 37,
'wal_level': 'replica',
'wal_keep_segments': None
},
'use_pg_rewind': None
}
}
},
'postgresql': {
'connect_address': f'{self._IP}:bar',
'listen': '6.6.6.6:1984',
'data_dir': 'data',
'bin_dir': '/bin/dir/from/running',
'parameters': {
'archive_command': 'my archive command',
'hba_file': os.path.join('data', 'pg_hba.conf'),
'ident_file': os.path.join('data', 'pg_ident.conf'),
'password_encryption': None
},
'authentication': {
'superuser': {
'username': 'foobar',
'password': 'qwerty',
'channel_binding': 'prefer',
'gssencmode': 'prefer',
'sslmode': 'prefer'
},
'replication': {
'username': self.no_value_msg,
'password': self.no_value_msg
},
'rewind': None
},
}
}
patch_config(self.config, conf)
def _get_running_instance_open_res(self):
hba_content = '\n'.join(self.config['postgresql']['pg_hba'] + ['#host all all all md5',
' host all all all md5',
'',
'hostall all all md5'])
ident_content = '\n'.join(['# something very interesting', ' '])
self.config['postgresql']['pg_hba'] += ['host all all all md5']
return [
mock_open(read_data=hba_content)(),
mock_open(read_data=ident_content)(),
mock_open(read_data='1984')(),
mock_open()()
]
@patch('os.makedirs')
@patch('yaml.safe_dump')
def test_generate_sample_config_pre_13_dir_creation(self, mock_config_dump, mock_makedir):
with patch('sys.argv', ['patroni.py', '--generate-sample-config', '/foo/bar.yml']), \
patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 9.4.3")) as pg_bin_mock, \
self.assertRaises(SystemExit) as e:
_main()
self.assertEqual(e.exception.code, 0)
self.assertEqual(self.config, mock_config_dump.call_args[0][0])
mock_makedir.assert_called_once()
pg_bin_mock.assert_called_once_with([os.path.join(self.environ['PATRONI_POSTGRESQL_BIN_DIR'],
self.environ['PATRONI_POSTGRESQL_BIN_POSTGRES']),
'--version'])
@patch('os.makedirs', Mock())
@patch('yaml.safe_dump')
def test_generate_sample_config_16(self, mock_config_dump):
conf = {
'bootstrap': {
'dcs': {
'postgresql': {
'parameters': {
'wal_keep_size': '128MB',
'wal_keep_segments': None
},
}
}
},
'postgresql': {
'parameters': {
'password_encryption': 'scram-sha-256'
},
'pg_hba': ['host all all all scram-sha-256',
f'host replication {self.environ["PATRONI_REPLICATION_USERNAME"]} all scram-sha-256'],
'authentication': {
'rewind': {
'username': self.environ['PATRONI_REWIND_USERNAME'],
'password': self.no_value_msg}
},
}
}
patch_config(self.config, conf)
with patch('sys.argv', ['patroni.py', '--generate-sample-config', '/foo/bar.yml']), \
self.assertRaises(SystemExit) as e:
_main()
self.assertEqual(e.exception.code, 0)
self.assertEqual(self.config, mock_config_dump.call_args[0][0])
@patch('os.makedirs', Mock())
@patch('yaml.safe_dump')
def test_generate_config_running_instance_16(self, mock_config_dump):
self._set_running_instance_config_vals()
with patch('builtins.open', Mock(side_effect=self._get_running_instance_open_res())), \
patch('sys.argv', ['patroni.py', '--generate-config',
'--dsn', 'host=foo port=bar user=foobar password=qwerty']), \
self.assertRaises(SystemExit) as e:
_main()
self.assertEqual(e.exception.code, 0)
self.assertEqual(self.config, mock_config_dump.call_args[0][0])
@patch('os.makedirs', Mock())
@patch('yaml.safe_dump')
def test_generate_config_running_instance_16_connect_from_env(self, mock_config_dump):
self._set_running_instance_config_vals()
# su auth params and connect host from env
os.environ['PGCHANNELBINDING'] = \
self.config['postgresql']['authentication']['superuser']['channel_binding'] = 'disable'
conf = {
'scope': 'my_cluster',
'bootstrap': {
'dcs': {
'postgresql': {
'parameters': {
'max_connections': 42,
'max_locks_per_transaction': 73,
'max_replication_slots': 21,
'max_wal_senders': 37,
'wal_level': 'replica',
'wal_keep_segments': None
},
'use_pg_rewind': None
}
}
},
'postgresql': {
'connect_address': f'{self._IP}:1984',
'authentication': {
'superuser': {
'username': self.environ['PGUSER'],
'password': self.environ['PGPASSWORD'],
'gssencmode': None,
'sslmode': None
},
},
}
}
patch_config(self.config, conf)
with patch('builtins.open', Mock(side_effect=self._get_running_instance_open_res())), \
patch('sys.argv', ['patroni.py', '--generate-config']), \
patch.object(MockConnect, 'server_version', PropertyMock(return_value=160000)), \
self.assertRaises(SystemExit) as e:
_main()
self.assertEqual(e.exception.code, 0)
self.assertEqual(self.config, mock_config_dump.call_args[0][0])
def test_generate_config_running_instance_errors(self):
# 1. Wrong DSN format
with patch('sys.argv', ['patroni.py', '--generate-config', '--dsn', 'host:foo port:bar user:foobar']), \
self.assertRaises(SystemExit) as e:
_main()
self.assertIn('Failed to parse DSN string', e.exception.code)
# 2. User is not a superuser
with patch('sys.argv', ['patroni.py',
'--generate-config', '--dsn', 'host=foo port=bar user=foobar password=pwd_from_dsn']), \
patch.object(MockCursor, 'rowcount', PropertyMock(return_value=0), create=True), \
patch.object(MockConnectionInfo, 'parameter_status', Mock(return_value='off')), \
self.assertRaises(SystemExit) as e:
_main()
self.assertIn('The provided user does not have superuser privilege', e.exception.code)
# 3. Error while calling postgres --version
with patch('subprocess.check_output', Mock(side_effect=OSError)), \
patch('sys.argv', ['patroni.py', '--generate-sample-config']), \
self.assertRaises(SystemExit) as e:
_main()
self.assertIn('Failed to get postgres version:', e.exception.code)
with patch('sys.argv', ['patroni.py', '--generate-config']):
# 4. empty postmaster.pid
with patch('builtins.open', Mock(side_effect=[mock_open(read_data='hba_content')(),
mock_open(read_data='ident_content')(),
mock_open(read_data='')()])), \
self.assertRaises(SystemExit) as e:
_main()
self.assertIn('Failed to obtain postmaster pid from postmaster.pid file', e.exception.code)
# 5. Failed to open postmaster.pid
with patch('builtins.open', Mock(side_effect=[mock_open(read_data='hba_content')(),
mock_open(read_data='ident_content')(),
OSError])), \
self.assertRaises(SystemExit) as e:
_main()
self.assertIn('Error while reading postmaster.pid file', e.exception.code)
# 6. Invalid postmaster pid
with patch('builtins.open', Mock(side_effect=[mock_open(read_data='hba_content')(),
mock_open(read_data='ident_content')(),
mock_open(read_data='1984')()])), \
patch('psutil.Process.__init__', Mock(return_value=None)), \
patch('psutil.Process.exe', Mock(side_effect=psutil.NoSuchProcess(1984))), \
self.assertRaises(SystemExit) as e:
_main()
self.assertIn("Obtained postmaster pid doesn't exist", e.exception.code)
# 7. Failed to open pg_hba
with patch('builtins.open', Mock(side_effect=OSError)), \
self.assertRaises(SystemExit) as e:
_main()
self.assertIn('Failed to read pg_hba.conf', e.exception.code)
# 8. Failed to open pg_ident
with patch('builtins.open', Mock(side_effect=[mock_open(read_data='hba_content')(), OSError])), \
self.assertRaises(SystemExit) as e:
_main()
self.assertIn('Failed to read pg_ident.conf', e.exception.code)
# 9. Failed PG connecttion
from . import psycopg
with patch('patroni.psycopg.connect', side_effect=psycopg.Error), \
self.assertRaises(SystemExit) as e:
_main()
self.assertIn('Failed to establish PostgreSQL connection', e.exception.code)
# 10. An unexpected error
with patch.object(AbstractConfigGenerator, '__init__', side_effect=psycopg.Error), \
self.assertRaises(SystemExit) as e:
_main()
self.assertIn('Unexpected exception', e.exception.code)
def test_get_address(self):
with patch('socket.getaddrinfo', Mock(side_effect=Exception)), \
patch('logging.warning') as mock_warning:
self.assertEqual(get_address(), (self.no_value_msg, self.no_value_msg))
self.assertIn('Failed to obtain address: %r', mock_warning.call_args_list[0][0])
tests/test_consul.py
+3 -2
View File
@@ -197,9 +197,10 @@ class TestConsul(unittest.TestCase):
@patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
def test_delete_leader(self):
self.c.delete_leader()
leader = self.c.get_cluster().leader
self.c.delete_leader(leader)
self.c._name = 'other'
self.c.delete_leader()
self.c.delete_leader(leader)
@patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
def test_initialize(self):
tests/test_etcd.py
+1 -1
View File
@@ -313,7 +313,7 @@ class TestEtcd(unittest.TestCase):
self.assertFalse(self.etcd.cancel_initialization())
def test_delete_leader(self):
self.assertFalse(self.etcd.delete_leader())
self.assertFalse(self.etcd.delete_leader(self.etcd.get_cluster().leader))
def test_delete_cluster(self):
self.assertFalse(self.etcd.delete_cluster())
tests/test_etcd3.py
+4 -3
View File
@@ -298,9 +298,10 @@ class TestEtcd3(BaseTestEtcd3):
self.etcd3.cancel_initialization()
def test_delete_leader(self):
self.etcd3.delete_leader()
leader = self.etcd3.get_cluster().leader
self.etcd3.delete_leader(leader)
self.etcd3._name = 'other'
self.etcd3.delete_leader()
self.etcd3.delete_leader(leader)
def test_delete_cluster(self):
self.etcd3.delete_cluster()
@@ -312,7 +313,7 @@ class TestEtcd3(BaseTestEtcd3):
self.etcd3.set_sync_state_value('', 1)
def test_delete_sync_state(self):
self.etcd3.delete_sync_state()
self.etcd3.delete_sync_state('1')
def test_watch(self):
self.etcd3.set_ttl(10)
tests/test_file_perm.py
+33
View File
@@ -0,0 +1,33 @@
import unittest
import stat
from mock import Mock, patch
from patroni.file_perm import pg_perm
class TestFilePermissions(unittest.TestCase):
@patch('os.stat')
@patch('os.umask')
@patch('patroni.file_perm.logger.error')
def test_set_umask(self, mock_logger, mock_umask, mock_stat):
mock_umask.side_effect = Exception
mock_stat.return_value.st_mode = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP
pg_perm.set_permissions_from_data_directory('test')
# umask is called with PG_MODE_MASK_GROUP
self.assertEqual(mock_umask.call_args[0][0], stat.S_IWGRP | stat.S_IRWXO)
self.assertEqual(mock_logger.call_args[0][0], 'Can not set umask to %03o: %r')
mock_umask.reset_mock()
mock_stat.return_value.st_mode = stat.S_IRWXU
pg_perm.set_permissions_from_data_directory('test')
# umask is called with PG_MODE_MASK_OWNER (permissions changed from group to owner)
self.assertEqual(mock_umask.call_args[0][0], stat.S_IRWXG | stat.S_IRWXO)
@patch('os.stat', Mock(side_effect=FileNotFoundError))
@patch('patroni.file_perm.logger.error')
def test_set_permissions_from_data_directory(self, mock_logger):
pg_perm.set_permissions_from_data_directory('test')
self.assertEqual(mock_logger.call_args[0][0], 'Can not check permissions on %s: %r')
tests/test_ha.py
+51 -41
View File
@@ -162,7 +162,7 @@ def run_async(self, func, args=()):
@patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
@patch.object(Postgresql, 'is_leader', Mock(return_value=True))
@patch.object(Postgresql, 'is_primary', Mock(return_value=True))
@patch.object(Postgresql, 'timeline_wal_position', Mock(return_value=(1, 10, 1)))
@patch.object(Postgresql, '_cluster_info_state_get', Mock(return_value=10))
@patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
@@ -224,7 +224,7 @@ class TestHa(PostgresInit):
@patch.object(Postgresql, 'received_timeline', Mock(return_value=None))
def test_touch_member(self):
self.p._major_version = 110000
self.p.is_leader = false
self.p.is_primary = false
self.p.timeline_wal_position = Mock(return_value=(0, 1, 0))
self.p.replica_cached_timeline = Mock(side_effect=Exception)
with patch.object(Postgresql, '_cluster_info_state_get', Mock(return_value='streaming')):
@@ -320,7 +320,7 @@ class TestHa(PostgresInit):
@patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
@patch.object(Rewind, 'can_rewind', PropertyMock(return_value=True))
def test_crash_recovery_before_rewind(self):
self.p.is_leader = false
self.p.is_primary = false
self.p.is_running = false
self.p.controldata = lambda: {'Database cluster state': 'in archive recovery',
'Database system identifier': SYSID}
@@ -365,7 +365,7 @@ class TestHa(PostgresInit):
@patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
def test_start_as_readonly(self):
self.p.is_leader = false
self.p.is_primary = false
self.p.is_healthy = true
self.ha.has_lock = true
self.p.controldata = lambda: {'Database cluster state': 'in production', 'Database system identifier': SYSID}
@@ -383,11 +383,11 @@ class TestHa(PostgresInit):
def test_promoted_by_acquiring_lock(self):
self.ha.is_healthiest_node = true
self.p.is_leader = false
self.p.is_primary = false
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
def test_promotion_cancelled_after_pre_promote_failed(self):
self.p.is_leader = false
self.p.is_primary = false
self.p._pre_promote = false
self.ha._is_healthiest_node = true
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
@@ -402,7 +402,7 @@ class TestHa(PostgresInit):
@patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
def test_long_promote(self):
self.ha.has_lock = true
self.p.is_leader = false
self.p.is_primary = false
self.p.set_role('primary')
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
@@ -413,7 +413,7 @@ class TestHa(PostgresInit):
def test_follow_new_leader_after_failing_to_obtain_lock(self):
self.ha.is_healthiest_node = true
self.ha.acquire_lock = false
self.p.is_leader = false
self.p.is_primary = false
self.assertEqual(self.ha.run_cycle(), 'following new leader after trying and failing to obtain lock')
def test_demote_because_not_healthiest(self):
@@ -422,21 +422,20 @@ class TestHa(PostgresInit):
def test_follow_new_leader_because_not_healthiest(self):
self.ha.is_healthiest_node = false
self.p.is_leader = false
self.p.is_primary = false
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
@patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
def test_promote_because_have_lock(self):
self.ha.has_lock = true
self.p.is_leader = false
self.p.is_primary = false
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader because I had the session lock')
def test_promote_without_watchdog(self):
self.ha.has_lock = true
self.p.is_leader = true
with patch.object(Watchdog, 'activate', Mock(return_value=False)):
self.assertEqual(self.ha.run_cycle(), 'Demoting self because watchdog could not be activated')
self.p.is_leader = false
self.p.is_primary = false
self.assertEqual(self.ha.run_cycle(), 'Not promoting self because watchdog could not be activated')
def test_leader_with_lock(self):
@@ -462,12 +461,12 @@ class TestHa(PostgresInit):
self.assertEqual(self.ha.run_cycle(), 'demoted self because failed to update leader lock in DCS')
with patch.object(Ha, '_get_node_to_follow', Mock(side_effect=DCSError('foo'))):
self.assertEqual(self.ha.run_cycle(), 'demoted self because failed to update leader lock in DCS')
self.p.is_leader = false
self.p.is_primary = false
self.assertEqual(self.ha.run_cycle(), 'not promoting because failed to update leader lock in DCS')
@patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
def test_follow(self):
self.p.is_leader = false
self.p.is_primary = false
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), a secondary, and following a leader ()')
self.ha.patroni.replicatefrom = "foo"
self.p.config.check_recovery_conf = Mock(return_value=(True, False))
@@ -484,13 +483,13 @@ class TestHa(PostgresInit):
def test_follow_in_pause(self):
self.ha.is_paused = true
self.assertEqual(self.ha.run_cycle(), 'PAUSE: continue to run as primary without lock')
self.p.is_leader = false
self.p.is_primary = false
self.assertEqual(self.ha.run_cycle(), 'PAUSE: no action. I am (postgresql0)')
@patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
@patch.object(Rewind, 'can_rewind', PropertyMock(return_value=True))
def test_follow_triggers_rewind(self):
self.p.is_leader = false
self.p.is_primary = false
self.ha._rewind.trigger_check_diverged_lsn()
self.ha.cluster = get_cluster_initialized_with_leader()
self.assertEqual(self.ha.run_cycle(), 'running pg_rewind from leader')
@@ -544,7 +543,7 @@ class TestHa(PostgresInit):
self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
self.ha.update_failsafe({'name': 'leader', 'api_url': 'http://127.0.0.1:8008/patroni',
'conn_url': 'postgres://127.0.0.1:5432/postgres', 'slots': {'foo': 1000}})
self.p.is_leader = false
self.p.is_primary = false
self.assertEqual(self.ha.run_cycle(), 'DCS is not accessible')
def test_no_dcs_connection_replica_failsafe_not_enabled_but_active(self):
@@ -552,7 +551,7 @@ class TestHa(PostgresInit):
self.ha.cluster = get_cluster_initialized_with_leader()
self.ha.update_failsafe({'name': 'leader', 'api_url': 'http://127.0.0.1:8008/patroni',
'conn_url': 'postgres://127.0.0.1:5432/postgres', 'slots': {'foo': 1000}})
self.p.is_leader = false
self.p.is_primary = false
self.assertEqual(self.ha.run_cycle(), 'DCS is not accessible')
def test_update_failsafe(self):
@@ -591,9 +590,9 @@ class TestHa(PostgresInit):
self.ha.cluster = get_cluster_not_initialized_without_leader()
self.e.initialize = true
self.assertEqual(self.ha.bootstrap(), 'trying to bootstrap a new cluster')
self.p.is_leader = false
self.p.is_primary = false
self.assertEqual(self.ha.run_cycle(), 'waiting for end of recovery after bootstrap')
self.p.is_leader = true
self.p.is_primary = true
self.ha.is_synchronous_mode = true
self.assertEqual(self.ha.run_cycle(), 'running post_bootstrap')
self.assertEqual(self.ha.run_cycle(), 'initialized a new cluster')
@@ -613,7 +612,6 @@ class TestHa(PostgresInit):
self.ha.cluster = get_cluster_not_initialized_without_leader()
self.e.initialize = true
self.ha.bootstrap()
self.p.is_leader = true
with patch.object(Watchdog, 'activate', Mock(return_value=False)), \
patch('patroni.ha.logger.error') as mock_logger:
self.assertEqual(self.ha.post_bootstrap(), 'running post_bootstrap')
@@ -745,10 +743,8 @@ class TestHa(PostgresInit):
self.assertEqual('PAUSE: no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
def test_manual_failover_from_leader_in_synchronous_mode(self):
self.p.is_leader = true
self.ha.has_lock = true
self.ha.is_synchronous_mode = true
self.ha.is_failover_possible = false
self.ha.process_sync_replication = Mock()
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None), (self.p.name, None))
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
@@ -757,7 +753,7 @@ class TestHa(PostgresInit):
self.assertEqual('manual failover: demoting myself', self.ha.run_cycle())
def test_manual_failover_process_no_leader(self):
self.p.is_leader = false
self.p.is_primary = false
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', self.p.name, None))
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None))
self.p.set_role('replica')
@@ -781,7 +777,7 @@ class TestHa(PostgresInit):
def test_manual_failover_process_no_leader_in_synchronous_mode(self):
self.ha.is_synchronous_mode = true
self.p.is_leader = false
self.p.is_primary = false
# switchover to a specific node, which name doesn't match our name (postgresql0)
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'other', None))
@@ -841,14 +837,14 @@ class TestHa(PostgresInit):
self.assertEqual(self.ha.run_cycle(), 'PAUSE: continue to run as primary without lock')
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'blabla', None))
self.assertEqual('PAUSE: acquired session lock as a leader', self.ha.run_cycle())
self.p.is_leader = false
self.p.is_primary = false
self.p.set_role('replica')
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', self.p.name, None))
self.assertEqual(self.ha.run_cycle(), 'PAUSE: promoted self to leader by acquiring session lock')
def test_is_healthiest_node(self):
self.ha.is_failsafe_mode = true
self.ha.state_handler.is_leader = false
self.p.is_primary = false
self.ha.patroni.nofailover = False
self.ha.fetch_node_status = get_node_status()
self.ha.dcs._last_failsafe = {'foo': ''}
@@ -862,7 +858,7 @@ class TestHa(PostgresInit):
self.assertFalse(self.ha.is_healthiest_node())
def test__is_healthiest_node(self):
self.p.is_leader = false
self.p.is_primary = false
self.ha.cluster = get_cluster_initialized_without_leader(sync=('postgresql1', self.p.name))
self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
@@ -961,7 +957,7 @@ class TestHa(PostgresInit):
self.assertTrue(self.ha.restart_matches("replica", "9.5.2", False))
def test_process_healthy_cluster_in_pause(self):
self.p.is_leader = false
self.p.is_primary = false
self.ha.is_paused = true
self.p.name = 'leader'
self.ha.cluster = get_cluster_initialized_with_leader()
@@ -972,7 +968,7 @@ class TestHa(PostgresInit):
@patch('patroni.postgresql.mtime', Mock(return_value=1588316884))
@patch('builtins.open', mock_open(read_data='1\t0/40159C0\tno recovery target specified\n'))
def test_process_healthy_standby_cluster_as_standby_leader(self):
self.p.is_leader = false
self.p.is_primary = false
self.p.name = 'leader'
self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
self.p.config.check_recovery_conf = Mock(return_value=(False, False))
@@ -984,7 +980,7 @@ class TestHa(PostgresInit):
self.assertEqual(self.ha.run_cycle(), 'promoted self to a standby leader because i had the session lock')
def test_process_healthy_standby_cluster_as_cascade_replica(self):
self.p.is_leader = false
self.p.is_primary = false
self.p.name = 'replica'
self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
self.assertEqual(self.ha.run_cycle(),
@@ -994,7 +990,7 @@ class TestHa(PostgresInit):
@patch.object(Cluster, 'is_unlocked', Mock(return_value=True))
def test_process_unhealthy_standby_cluster_as_standby_leader(self):
self.p.is_leader = false
self.p.is_primary = false
self.p.name = 'leader'
self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
self.ha.sysid_valid = true
@@ -1004,13 +1000,13 @@ class TestHa(PostgresInit):
@patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
@patch.object(Rewind, 'can_rewind', PropertyMock(return_value=True))
def test_process_unhealthy_standby_cluster_as_cascade_replica(self):
self.p.is_leader = false
self.p.is_primary = false
self.p.name = 'replica'
self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
self.assertTrue(self.ha.run_cycle().startswith('running pg_rewind from remote_member:'))
def test_recover_unhealthy_leader_in_standby_cluster(self):
self.p.is_leader = false
self.p.is_primary = false
self.p.name = 'leader'
self.p.is_running = false
self.p.follow = false
@@ -1019,7 +1015,7 @@ class TestHa(PostgresInit):
@patch.object(Cluster, 'is_unlocked', Mock(return_value=True))
def test_recover_unhealthy_unlocked_standby_cluster(self):
self.p.is_leader = false
self.p.is_primary = false
self.p.name = 'leader'
self.p.is_running = false
self.p.follow = false
@@ -1079,7 +1075,7 @@ class TestHa(PostgresInit):
check_calls([(update_lock, True), (demote, True)])
self.ha.has_lock = false
self.p.is_leader = false
self.p.is_primary = false
self.assertEqual(self.ha.run_cycle(),
'no action. I am (postgresql0), a secondary, and following a leader (leader)')
check_calls([(update_lock, False), (demote, False)])
@@ -1213,7 +1209,7 @@ class TestHa(PostgresInit):
self.ha.is_synchronous_mode = true
mock_set_sync = self.p.sync_handler.set_synchronous_standby_names = Mock()
self.p.is_leader = false
self.p.is_primary = false
self.p.set_role('replica')
self.ha.has_lock = true
mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
@@ -1236,7 +1232,7 @@ class TestHa(PostgresInit):
def test_unhealthy_sync_mode(self):
self.ha.is_synchronous_mode = true
self.p.is_leader = false
self.p.is_primary = false
self.p.set_role('replica')
self.p.name = 'other'
self.ha.cluster = get_cluster_initialized_without_leader(sync=('leader', 'other2'))
@@ -1267,7 +1263,7 @@ class TestHa(PostgresInit):
self.ha.is_synchronous_mode = true
self.p.name = 'other'
self.p.is_leader = false
self.p.is_primary = false
self.p.set_role('replica')
mock_restart = self.p.restart = Mock(return_value=True)
self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
@@ -1298,6 +1294,19 @@ class TestHa(PostgresInit):
mock_restart.assert_called_once()
self.ha.dcs.get_cluster.assert_not_called()
def test_enable_synchronous_mode(self):
self.ha.is_synchronous_mode = true
self.ha.has_lock = true
self.p.name = 'leader'
self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
with patch('patroni.ha.logger.info') as mock_logger:
self.ha.run_cycle()
self.assertEqual(mock_logger.call_args[0][0], 'Enabled synchronous replication')
self.ha.dcs.write_sync_state = Mock(return_value=None)
with patch('patroni.ha.logger.warning') as mock_logger:
self.ha.run_cycle()
self.assertEqual(mock_logger.call_args[0][0], 'Updating sync state failed')
def test_effective_tags(self):
self.ha._disable_sync = True
self.assertEqual(self.ha.get_effective_tags(), {'foo': 'bar', 'nosync': True})
@@ -1370,7 +1379,7 @@ class TestHa(PostgresInit):
@patch('sys.exit', return_value=1)
def test_abort_join(self, exit_mock):
self.ha.cluster = get_cluster_not_initialized_without_leader()
self.p.is_leader = false
self.p.is_primary = false
self.ha.run_cycle()
exit_mock.assert_called_once_with(1)
@@ -1421,6 +1430,7 @@ class TestHa(PostgresInit):
@patch('os.open', Mock())
@patch('os.fsync', Mock())
@patch('os.close', Mock())
@patch('os.chmod', Mock())
@patch('os.rename', Mock())
@patch('patroni.postgresql.Postgresql.is_starting', Mock(return_value=False))
@patch('builtins.open', mock_open())
@@ -1429,7 +1439,7 @@ class TestHa(PostgresInit):
@patch.object(SlotsHandler, 'sync_replication_slots', Mock(return_value=['ls']))
def test_follow_copy(self):
self.ha.cluster.config.data['slots'] = {'ls': {'database': 'a', 'plugin': 'b'}}
self.p.is_leader = false
self.p.is_primary = false
self.assertTrue(self.ha.run_cycle().startswith('Copying logical slots'))
def test_acquire_lock(self):
tests/test_kubernetes.py
+7 -7
View File
@@ -306,25 +306,25 @@ class TestKubernetesConfigMaps(BaseTestKubernetes):
self.k.touch_member({'state': 'running', 'role': 'replica'})
mock_patch_namespaced_pod.assert_called()
self.assertEqual(mock_patch_namespaced_pod.call_args.args[2].metadata.labels['isMaster'], 'false')
self.assertEqual(mock_patch_namespaced_pod.call_args.args[2].metadata.labels['tmp_role'], 'replica')
self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['isMaster'], 'false')
self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['tmp_role'], 'replica')
self.k.touch_member({'state': 'running', 'role': 'standby-leader'})
mock_patch_namespaced_pod.assert_called()
self.assertEqual(mock_patch_namespaced_pod.call_args.args[2].metadata.labels['isMaster'], 'false')
self.assertEqual(mock_patch_namespaced_pod.call_args.args[2].metadata.labels['tmp_role'], 'standby-leader')
self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['isMaster'], 'false')
self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['tmp_role'], 'standby-leader')
self.k._name = 'p-0'
self.k.touch_member({'role': 'primary'})
mock_patch_namespaced_pod.assert_called()
self.assertEqual(mock_patch_namespaced_pod.call_args.args[2].metadata.labels['isMaster'], 'true')
self.assertEqual(mock_patch_namespaced_pod.call_args.args[2].metadata.labels['tmp_role'], 'master')
self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['isMaster'], 'true')
self.assertEqual(mock_patch_namespaced_pod.call_args[0][2].metadata.labels['tmp_role'], 'master')
def test_initialize(self):
self.k.initialize()
def test_delete_leader(self):
self.k.delete_leader(1)
self.k.delete_leader(self.k.get_cluster().leader, 1)
def test_cancel_initialization(self):
self.k.cancel_initialization()
tests/test_patroni.py
+2
View File
@@ -40,6 +40,7 @@ class MockFrozenImporter(object):
@patch('time.sleep', Mock())
@patch('subprocess.call', Mock(return_value=0))
@patch('patroni.psycopg.connect', psycopg_connect)
@patch('urllib3.PoolManager.request', Mock(side_effect=Exception))
@patch.object(ConfigHandler, 'append_pg_hba', Mock())
@patch.object(ConfigHandler, 'write_postgresql_conf', Mock())
@patch.object(ConfigHandler, 'write_recovery_conf', Mock())
@@ -63,6 +64,7 @@ class TestPatroni(unittest.TestCase):
self.assertRaises(SystemExit, _main)
@patch('pkgutil.iter_importers', Mock(return_value=[MockFrozenImporter()]))
@patch('urllib3.PoolManager.request', Mock(side_effect=Exception))
@patch('sys.frozen', Mock(return_value=True), create=True)
@patch.object(HTTPServer, '__init__', Mock())
@patch.object(etcd.Client, 'read', etcd_read)
tests/test_postgresql.py
+15 -8
View File
@@ -346,8 +346,7 @@ class TestPostgresql(BaseTestPostgresql):
@patch.object(Postgresql, 'start', Mock())
def test_follow(self):
self.p.call_nowait(CallbackAction.ON_START)
m = RemoteMember.from_name_and_data('1', {'restore_command': '2', 'primary_slot_name': 'foo',
'conn_kwargs': {'host': 'bar'}})
m = RemoteMember('1', {'restore_command': '2', 'primary_slot_name': 'foo', 'conn_kwargs': {'host': 'bar'}})
self.p.follow(m)
with patch.object(Postgresql, 'ensure_major_version_is_known', Mock(return_value=False)):
self.assertIsNone(self.p.follow(m))
@@ -364,11 +363,11 @@ class TestPostgresql(BaseTestPostgresql):
self.assertRaises(psycopg.ProgrammingError, self.p.query, 'blabla')
@patch.object(Postgresql, 'pg_isready', Mock(return_value=STATE_REJECT))
def test_is_leader(self):
self.assertTrue(self.p.is_leader())
def test_is_primary(self):
self.assertTrue(self.p.is_primary())
self.p.reset_cluster_info_state(None)
with patch.object(Postgresql, '_query', Mock(side_effect=RetryFailedError(''))):
self.assertFalse(self.p.is_leader())
self.assertFalse(self.p.is_primary())
@patch.object(Postgresql, 'controldata', Mock(return_value={'Database cluster state': 'shut down',
'Latest checkpoint location': '0/1ADBC18',
@@ -462,7 +461,7 @@ class TestPostgresql(BaseTestPostgresql):
self.assertIsNone(self.p.call_nowait(CallbackAction.ON_START))
@patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
def test_is_leader_exception(self):
def test_is_primary_exception(self):
self.p.start()
self.p.query = Mock(side_effect=psycopg.OperationalError("not supported"))
self.assertTrue(self.p.stop())
@@ -523,8 +522,9 @@ class TestPostgresql(BaseTestPostgresql):
def test_save_configuration_files(self):
self.p.config.save_configuration_files()
@patch('os.path.isfile', Mock(side_effect=[False, True]))
@patch('shutil.copy', Mock(side_effect=IOError))
@patch('os.path.isfile', Mock(side_effect=[False, True, False, True]))
@patch('shutil.copy', Mock(side_effect=[None, IOError]))
@patch('os.chmod', Mock())
def test_restore_configuration_files(self):
self.p.config.restore_configuration_files()
@@ -955,3 +955,10 @@ class TestPostgresql2(BaseTestPostgresql):
gucs = self.p.available_gucs
self.assertIsInstance(gucs, CaseInsensitiveSet)
self.assertEqual(gucs, mock_available_gucs.return_value)
def test_cluster_info_query(self):
self.assertIn('diff(pg_catalog.pg_current_wal_flush_lsn(', self.p.cluster_info_query)
self.p._major_version = 90600
self.assertIn('diff(pg_catalog.pg_current_xlog_flush_location(', self.p.cluster_info_query)
self.p._major_version = 90500
self.assertIn('diff(pg_catalog.pg_current_xlog_location(', self.p.cluster_info_query)
tests/test_raft.py
+4 -4
View File
@@ -142,25 +142,25 @@ class TestRaft(unittest.TestCase):
raft._citus_group = '1'
self.assertTrue(raft.manual_failover('foo', 'bar'))
raft._citus_group = '0'
self.assertTrue(raft.take_leader())
cluster = raft.get_cluster()
self.assertIsInstance(cluster, Cluster)
self.assertIsInstance(cluster.workers[1], Cluster)
leader = cluster.leader
self.assertTrue(raft.delete_leader(leader))
self.assertTrue(raft._sync_obj.set(raft.status_path, '{"optime":1234567,"slots":{"ls":12345}}'))
leader = raft.get_cluster().leader
raft.get_cluster()
self.assertTrue(raft.update_leader(leader, '1', failsafe={'foo': 'bat'}))
self.assertTrue(raft._sync_obj.set(raft.failsafe_path, '{"foo"}'))
self.assertTrue(raft._sync_obj.set(raft.status_path, '{'))
raft.get_citus_coordinator()
self.assertTrue(raft.delete_sync_state())
self.assertTrue(raft.delete_leader())
self.assertTrue(raft.set_history_value(''))
self.assertTrue(raft.delete_cluster())
raft._citus_group = '1'
self.assertTrue(raft.delete_cluster())
raft._citus_group = None
raft.get_cluster()
self.assertTrue(raft.take_leader())
raft.get_cluster()
raft.watch(None, 0.001)
raft._sync_obj.destroy()
tests/test_rewind.py
+2 -2
View File
@@ -253,8 +253,8 @@ class TestRewind(BaseTestPostgresql):
mock_logger_info.call_args[0])
mock_logger_info.reset_mock()
mock_subprocess_call.assert_called_once()
self.assertEqual(mock_subprocess_call.call_args.args[0], ['command 000000000000000000000000'])
self.assertEqual(mock_subprocess_call.call_args.kwargs['shell'], True)
self.assertEqual(mock_subprocess_call.call_args[0][0], ['command 000000000000000000000000'])
self.assertEqual(mock_subprocess_call.call_args[1]['shell'], True)
mock_subprocess_call.reset_mock()
# failed archive_command call
tests/test_slots.py
+21 -4
View File
@@ -48,7 +48,7 @@ class TestSlotsHandler(BaseTestPostgresql):
self.s.sync_replication_slots(cluster, False)
mock_debug.assert_called_once()
self.p.set_role('replica')
with patch.object(Postgresql, 'is_leader', Mock(return_value=False)), \
with patch.object(Postgresql, 'is_primary', Mock(return_value=False)), \
patch.object(SlotsHandler, 'drop_replication_slot') as mock_drop:
self.s.sync_replication_slots(cluster, False, paused=True)
mock_drop.assert_not_called()
@@ -67,6 +67,23 @@ class TestSlotsHandler(BaseTestPostgresql):
with patch.object(Postgresql, 'major_version', PropertyMock(return_value=90618)):
self.s.sync_replication_slots(cluster, False)
def test_cascading_replica_sync_replication_slots(self):
"""Test sync with a cascading replica so physical slots are present on a replica."""
config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}}}, 1)
cascading_replica = Member(0, 'test-2', 28, {
'state': 'running', 'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
'tags': {'replicatefrom': 'postgresql0'}
})
cluster = Cluster(True, config, self.leader, 0,
[self.me, self.other, self.leadermem, cascading_replica],
None, SyncState.empty(), None, {'ls': 10}, None)
self.p.set_role('replica')
with patch.object(Postgresql, '_query') as mock_query, \
patch.object(Postgresql, 'is_primary', Mock(return_value=False)):
mock_query.return_value = [('ls', 'logical', 'b', 'a', 5, 12345, 105)]
ret = self.s.sync_replication_slots(cluster, False)
self.assertEqual(ret, [])
def test_process_permanent_slots(self):
config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}},
'ignore_slots': [{'name': 'blabla'}]}, 1)
@@ -89,7 +106,7 @@ class TestSlotsHandler(BaseTestPostgresql):
"confirmed_flush_lsn": 12345, "catalog_xmin": 105}])
self.assertEqual(self.p.slots(), {})
@patch.object(Postgresql, 'is_leader', Mock(return_value=False))
@patch.object(Postgresql, 'is_primary', Mock(return_value=False))
def test__ensure_logical_slots_replica(self):
self.p.set_role('replica')
self.cluster.slots['ls'] = 12346
@@ -116,7 +133,7 @@ class TestSlotsHandler(BaseTestPostgresql):
@patch.object(Postgresql, 'stop', Mock(return_value=True))
@patch.object(Postgresql, 'start', Mock(return_value=True))
@patch.object(Postgresql, 'is_leader', Mock(return_value=False))
@patch.object(Postgresql, 'is_primary', Mock(return_value=False))
def test_check_logical_slots_readiness(self):
self.s.copy_logical_slots(self.cluster, ['ls'])
with patch.object(MockCursor, '__iter__', Mock(return_value=iter([('postgresql0', None)]))), \
@@ -130,7 +147,7 @@ class TestSlotsHandler(BaseTestPostgresql):
@patch.object(Postgresql, 'stop', Mock(return_value=True))
@patch.object(Postgresql, 'start', Mock(return_value=True))
@patch.object(Postgresql, 'is_leader', Mock(return_value=False))
@patch.object(Postgresql, 'is_primary', Mock(return_value=False))
def test_on_promote(self):
self.s.schedule_advance_slots({'foo': {'bar': 100}})
self.s.copy_logical_slots(self.cluster, ['ls'])
tests/test_zookeeper.py
+1 -1
View File
@@ -202,7 +202,7 @@ class TestZooKeeper(unittest.TestCase):
mock_logger.assert_called_once()
def test_delete_leader(self):
self.assertTrue(self.zk.delete_leader())
self.assertTrue(self.zk.delete_leader(self.zk.get_cluster().leader))
def test_set_failover_value(self):
self.zk.set_failover_value('')