From 895eefaa51610d61f65259db6dabc06667f2b4db Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Wed, 19 Jul 2017 12:25:50 +0200 Subject: [PATCH 1/4] Document bootstrapping and replica creation (#478) Describe parameters around custom replica creation and bootstrap --- docs/SETTINGS.rst | 10 +++- docs/index.rst | 1 + docs/replica_bootstrap.rst | 98 ++++++++++++++++++++++++++++++++++++++ docs/replication_modes.rst | 2 +- 4 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 docs/replica_bootstrap.rst diff --git a/docs/SETTINGS.rst b/docs/SETTINGS.rst index fac8a90f..84265de9 100644 --- a/docs/SETTINGS.rst +++ b/docs/SETTINGS.rst @@ -23,6 +23,10 @@ Bootstrap configuration - **use\_slots**: whether or not to use replication_slots. Must be False for PostgreSQL 9.3. You should comment out max_replication_slots before it becomes ineligible for leader status. - **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower. - **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work. +- **method**: custom script to use for bootstrpapping this cluster. + See :ref:`custom bootstrap methods documentation ` for details. + When ``initdb`` is specified revert to the default ``initdb`` command. ``initdb`` is also triggered when no ``method`` + parameter is present in the configuration file. - **initdb**: List options to be passed on to initdb. - **- data-checksums**: Must be enabled when pg_rewind is needed on 9.3. - **- encoding: UTF8**: default encoding for new databases. @@ -80,7 +84,9 @@ PostgreSQL - **on\_start**: run this script when the cluster starts. - **on\_stop**: run this script when the cluster stops. - **connect\_address**: IP address + port through which Postgres is accessible from other nodes and applications. -- **create\_replica\_methods**: an ordered list of the create methods for turning a Patroni node into a new replica. "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its own config item. +- **create\_replica\_method**: an ordered list of the create methods for turning a Patroni node into a new replica. + "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its + own config item. See :ref:`custom replica creation methods documentation ` for further explanation. - **data\_dir**: The location of the Postgres data directory, either existing or to be initialized by Patroni. - **config\_dir**: The location of the Postgres configuration directory, defaults to the data directory. Must be writable by Patroni. - **bin\_dir**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The default value is an empty string meaning that PATH environment variable will be used to find the executables. @@ -90,7 +96,7 @@ PostgreSQL - **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower. - **custom\_conf** : path to an optional custom ``postgresql.conf`` file, that will be used in place of ``postgresql.base.conf``. The file must exist on all cluster nodes, be readable by PostgreSQL and will be included from its location on the real ``postgresql.conf``. Note that Patroni will not monitor this file for changes, nor backup it. However, its settings can still be overriden by Patroni's own configuration facilities - see `dynamic configuration `__ for details. - **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work. -- **pg\_hba**: list of lines that Patroni will use to generate ``pg_hba.conf``. This parameter has higher priority than ``bootstrap.pg_hba``. Together with `dynamic configuration `__ it simplifies management of ``pg_hba.conf``. +- **pg\_hba**: list of lines that Patroni will use to generate ``pg_hba.conf``. This parameter has higher priority than ``bootstrap.pg_hba``. Together with :ref:`dynamic configuration ` it simplifies management of ``pg_hba.conf``. - **- host all all 0.0.0.0/0 md5**. - **- host replication replicator 127.0.0.1/32 md5**: A line like this is required for replication. - **pg\_ctl\_timeout**: How long should pg_ctl wait when doing ``start``, ``stop`` or ``restart``. Default value is 60 seconds. diff --git a/docs/index.rst b/docs/index.rst index 6764b30d..f46babb9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,6 +21,7 @@ We call Patroni a "template" because it is far from being a one-size-fits-all or dynamic_configuration ENVIRONMENT SETTINGS + replica_bootstrap replication_modes pause releases diff --git a/docs/replica_bootstrap.rst b/docs/replica_bootstrap.rst new file mode 100644 index 00000000..1aecf629 --- /dev/null +++ b/docs/replica_bootstrap.rst @@ -0,0 +1,98 @@ +Replica imaging and bootstrap +============================= + +Patroni allows customizing creation of a new replica. It also supports defining what happens when the new empty cluster +is being bootstrapped. The distinction between two is well defined: Patroni creates replicas only if the ``initialize`` +key is present in Etcd for the cluster. If there is no ``initialize`` key - Patroni calls bootstrap exclusively on the +first node that takes the initialize key lock. + +.. _custom_bootstrap: + +Bootstrap +--------- + +PostgreSQL provides ``initdb`` command to initialize a new cluster and Patroni calls it by default. In certain cases, +particularly when creating a new cluster as a copy of an existing one, it is necessary to replace a built-in method with +custom actions. Patroni supports executing user-defined scripts to bootstrap new clusters, supplying some required +arguments to them, i.e. the name of the cluster and the path to the data directory. This is configured in the +``bootstrap`` section of the Patroni configuration. For example: + +.. code:: YAML + + bootstrap: + method: + : + command: [param1 [, ...]] + recovery_conf: + recovery_target_action: promote + recovery_target_timeline: latest + restore_command: + + +Each bootstrap method must define at least a ``name`` and a ``command``. A special ``initdb`` method is available to trigger +the default behavior, in which case ``method`` parameter can be omitted altogether. The ``command`` can be specified using either +an absolute path, or the one relative to the ``patroni`` command location. In addition to the fixed parameters defined +in the configuration files, Patroni supplies two cluster-specific ones: + +--scope + Name of the cluster to be bootstrapped +--datadir + Path to the data directory of the cluster instance to be bootstrapped + +If the bootstrap script returns 0, Patroni tries to configure and start the PostgreSQL instance produced by it. If any +of the intermediate steps fail, or the script returns a non-zero value, Patroni assumes that the bootstrap has failed, +cleans up after itself and releases the initialize lock to give another node the opportunity to bootstrap. + +If a ``recovery_conf`` block is defined in the same section as the custom bootstrap method, Patroni will generate a +``recovery.conf`` before starting the newly bootstrapped instance. Typically, such recovery.conf should contain at least +one of the ``recovery_target_*`` parameters, together with the ``recovery_target_timeline`` set to ``promote``. + + .. note:: Bootstrap methods are neither chained, nor fallen-back to the default one in case the primary one fails + + +.. _custom_replica_creation: + +Building replicas +----------------- + +Patroni uses tried and proven ``pg_basebackup`` in order to create new replicas. One downside of it is that it requires +a running master node. Another one is the lack of 'on-the-fly' compression for the backup data and no built-in cleanup +for outdated backup files. Some people prefer other backup solutions, such as ``WAL-E``, ``pgBackRest``, ``Barman`` and +others, or simply roll their own scripts. In order to accommodate all those use-cases Patroni supports running custom +scripts to clone a new replica. Those are configured in the ``postgresql`` configuration block: + +.. code:: YAML + + postgresql: + create_replica_method: + - wal_e + - basebackup + wal_e: + command: patroni_wale_restore + no_master: 1 + envdir: {{WALE_ENV_DIR}} + use_iam: 1 + + +The ``create_replica_method`` defines available replica creation methods and the order of executing them. Patroni will +stop on the first one that returns 0. The basebackup is the built-in method and doesn't require any configuration. The +rest of the methods should define a separate section in the configuration file, listing the command to execute and any +custom parameters that should be passed to that command. All parameters will be passed in a ``--name=value`` format. +Besides user-defined parameters, Patroni supplies a couple of cluster-specific ones: + +--scope + Which cluster this replica belongs to +--datadir + Path to the data directory of the replica +--role + Always 'replica' +--connstring + Connection string to connect to the cluster member to clone from (master or other replica). The user in the + connection string can execute SQL and replication protocol commands. + +A special ``no_master`` parameter, if defined, allows Patroni to call the replica creation method even if there is no +running master or replicas. In that case, an empty string will be passed in a connection string. This is useful for +restoring the formerly running cluster from the binary backup. + +If all replica creation methods fail, Patroni will try again all methods in order during the next event loop cycle. + diff --git a/docs/replication_modes.rst b/docs/replication_modes.rst index 06acbce3..805735d7 100644 --- a/docs/replication_modes.rst +++ b/docs/replication_modes.rst @@ -44,7 +44,7 @@ When ``synchronous_mode`` is on and a standby crashes, commits will block until You can ensure that a standby never becomes the synchronous standby by setting ``nosync`` tag to true. This is recommended to set for standbys that are behind slow network connections and would cause performance degradation when becoming a synchronous standby. -Synchronous mode can be switched on and off via Patroni REST interface. See `dynamic configuration `__ for instructions. +Synchronous mode can be switched on and off via Patroni REST interface. See :ref:`dynamic configuration ` for instructions. Synchronous mode implementation From 7e066a18cbe3537fa8e71cd4805d95de331dfe78 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Sat, 22 Jul 2017 09:45:28 +0200 Subject: [PATCH 2/4] Install locales package and define default locale as en_US.UTF-8 (#480) otherwise initdb fails --- Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8d99b11b..d412ebd9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,10 @@ RUN export DEBIAN_FRONTEND=noninteractive \ && apt-get update -y \ && apt-get upgrade -y \ && apt-get install -y curl jq haproxy python-psycopg2 python-yaml python-requests python-six python-pysocks \ - python-dateutil python-pip python-setuptools python-prettytable python-wheel python-psutil python \ + python-dateutil python-pip python-setuptools python-prettytable python-wheel python-psutil python locales \ + + ## Make sure we have a en_US.UTF-8 locale available + && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ && pip install 'python-etcd>=0.4.3,<0.5' click tzlocal cdiff \ @@ -40,5 +43,6 @@ RUN mkdir /data/ && touch /pgpass /patroni.yml \ EXPOSE 2379 5432 8008 +ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 ENTRYPOINT ["/bin/bash", "/entrypoint.sh"] USER postgres From cb360f089c0ae8eaa1b732b221033cba2bf29af9 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Sat, 22 Jul 2017 09:46:05 +0200 Subject: [PATCH 3/4] Restart postgres after custom bootstrap if hba_file is defined in configuration (#482) In addition to that always use absolute paths to config files. Fixes https://github.com/zalando/patroni/issues/481 --- patroni/postgresql.py | 53 +++++++++++++++++++++++----------------- tests/test_postgresql.py | 37 ++++++++++++++++------------ 2 files changed, 52 insertions(+), 38 deletions(-) diff --git a/patroni/postgresql.py b/patroni/postgresql.py index 02f203df..7b37b39c 100644 --- a/patroni/postgresql.py +++ b/patroni/postgresql.py @@ -102,7 +102,7 @@ class Postgresql(object): self._bin_dir = config.get('bin_dir') or '' self._database = config.get('database', 'postgres') self._data_dir = config['data_dir'] - self._config_dir = config.get('config_dir') or self._data_dir + self._config_dir = os.path.abspath(config.get('config_dir') or self._data_dir) self._pending_restart = False self._running_custom_bootstrap = False self.__thread_ident = current_thread().ident @@ -170,9 +170,9 @@ class Postgresql(object): configuration = [os.path.basename(self._postgresql_conf)] if 'custom_conf' not in self.config: configuration.append(os.path.basename(self._postgresql_base_conf)) - if not self.config['parameters'].get('hba_file'): + if not self._server_parameters.get('hba_file'): configuration.append('pg_hba.conf') - if not self.config['parameters'].get('ident_file'): + if not self._server_parameters.get('ident_file'): configuration.append('pg_ident.conf') return configuration @@ -226,8 +226,12 @@ class Postgresql(object): parameters['synchronous_standby_names'] = self._synchronous_standby_names if self._major_version >= 90600 and parameters['wal_level'] == 'hot_standby': parameters['wal_level'] = 'replica' - return {k: v for k, v in parameters.items() if not self._major_version or - self._major_version >= self.CMDLINE_OPTIONS.get(k, (0, 1, 90100))[2]} + ret = {k: v for k, v in parameters.items() if not self._major_version or + self._major_version >= self.CMDLINE_OPTIONS.get(k, (0, 1, 90100))[2]} + for k in ('hba_file', 'ident_file'): + if k in ret: + ret[k] = os.path.join(self._config_dir, ret[k]) + return ret def resolve_connection_addresses(self): port = self._server_parameters['port'] @@ -335,7 +339,7 @@ class Postgresql(object): conf_changed = True break - if not config['parameters'].get('hba_file') and config.get('pg_hba'): + if not server_parameters.get('hba_file') and config.get('pg_hba'): hba_changed = self.config.get('pg_hba', []) != config['pg_hba'] self.config = config @@ -519,7 +523,7 @@ class Postgresql(object): if pwfile: os.remove(pwfile) if ret: - if not self.config['parameters'].get('hba_file') and not self.config.get('pg_hba'): + if not self._server_parameters.get('hba_file') and not self.config.get('pg_hba'): self.write_pg_hba(config.get('pg_hba', [])) self._major_version = self.get_major_version() self._server_parameters = self.get_server_parameters(self.config) @@ -1085,7 +1089,6 @@ class Postgresql(object): with open(self._postgresql_conf, 'w') as f: f.write(self._CONFIG_WARNING_HEADER) f.write("include '{0}'\n\n".format(self.config.get('custom_conf') or self._postgresql_base_conf_name)) - f.write("data_directory = '{0}'\n".format(self._data_dir)) for name, value in sorted(self._server_parameters.items()): if not self._running_custom_bootstrap or name != 'hba_file': f.write("{0} = '{1}'\n".format(name, value)) @@ -1130,7 +1133,7 @@ class Postgresql(object): for address, t in addresses.items(): f.write('{0}\t{1}\t{2}\t{3}\ttrust\n'.format(t, self._database, self._superuser.get('username') or 'all', address)) - elif not self.config['parameters'].get('hba_file') and self.config.get('pg_hba'): + elif not self._server_parameters.get('hba_file') and self.config.get('pg_hba'): with open(self._pg_hba_conf, 'w') as f: f.write(self._CONFIG_WARNING_HEADER) for line in self.config['pg_hba']: @@ -1538,20 +1541,6 @@ $$""".format(name, ' '.join(options)), name, password, password) try: self.create_or_update_role(self._superuser['username'], self._superuser['password'], ['SUPERUSER']) - # We were doing a custom bootstrap instead of running initdb, therefore we opened trust - # access from certain addresses to be able to reach cluster and change password - if self._running_custom_bootstrap: - self._running_custom_bootstrap = False - # If we don't have custom configuration for pg_hba.conf we need to restore original file - if not self.config.get('pg_hba'): - os.unlink(self._pg_hba_conf) - self.restore_configuration_files() - self._write_postgresql_conf() - self._replace_pg_hba() - self.reload() - time.sleep(1) # give a time to postgres to "reload" configuration files - self.close_connection() # close connection to reconnect with a new password - task.complete(self.run_bootstrap_post_init(config)) if task.result: self.create_or_update_role(self._replication['username'], @@ -1559,6 +1548,24 @@ $$""".format(name, ' '.join(options)), name, password, password) for name, value in (config.get('users') or {}).items(): if name not in (self._superuser.get('username'), self._replication['username']): self.create_or_update_role(name, value['password'], value.get('options', [])) + + # We were doing a custom bootstrap instead of running initdb, therefore we opened trust + # access from certain addresses to be able to reach cluster and change password + if self._running_custom_bootstrap: + self._running_custom_bootstrap = False + # If we don't have custom configuration for pg_hba.conf we need to restore original file + if not self.config.get('pg_hba'): + os.unlink(self._pg_hba_conf) + self.restore_configuration_files() + self._write_postgresql_conf() + if self._server_parameters.get('hba_file') and \ + self._server_parameters['hba_file'] != self._pg_hba_conf: + self.restart() + else: + self._replace_pg_hba() + self.reload() + time.sleep(1) # give a time to postgres to "reload" configuration files + self.close_connection() # close connection to reconnect with a new password except Exception: logger.exception('post_bootstrap') task.complete(False) diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py index 0dd82344..4b1342df 100644 --- a/tests/test_postgresql.py +++ b/tests/test_postgresql.py @@ -280,7 +280,7 @@ class TestPostgresql(unittest.TestCase): self.assertFalse(self.p.stop()) mock_get_pid.return_value = 123 with patch('os.kill', Mock(side_effect=[OSError(errno.ESRCH, ''), OSError, None])),\ - patch('psutil.Process', Mock(side_effect=psutil.NoSuchProcess(123))): + patch('psutil.Process', Mock(side_effect=psutil.NoSuchProcess(123))): self.assertTrue(self.p.stop()) self.assertFalse(self.p.stop()) self.p.stop_safepoint_reached.clear() @@ -542,7 +542,7 @@ class TestPostgresql(unittest.TestCase): patch('os.unlink', Mock()),\ patch.object(Postgresql, 'save_configuration_files', Mock()),\ patch.object(Postgresql, 'restore_configuration_files', Mock()),\ - patch.object(Postgresql, 'write_recovery_conf', Mock()): + patch.object(Postgresql, 'write_recovery_conf', Mock()): with self.assertRaises(Exception) as e: self.p.bootstrap(config) self.assertEqual(str(e.exception), '42') @@ -554,24 +554,31 @@ class TestPostgresql(unittest.TestCase): self.assertEqual(str(e.exception), '42') @patch('time.sleep', Mock()) - @patch.object(Postgresql, 'run_bootstrap_post_init', Mock(side_effect=Exception)) + @patch('os.unlink', Mock()) + @patch.object(Postgresql, 'run_bootstrap_post_init', Mock(return_value=True)) + @patch.object(Postgresql, '_custom_bootstrap', Mock(return_value=True)) + @patch.object(Postgresql, 'start', Mock(return_value=True)) def test_post_bootstrap(self): config = {'method': 'foo', 'foo': {'command': 'bar'}} - with patch('subprocess.call', Mock(return_value=0)), \ - patch('subprocess.Popen', Mock(side_effect=Exception("42"))), \ - patch('os.path.isfile', Mock(return_value=True)),\ - patch('os.unlink', Mock()), \ - patch.object(Postgresql, 'save_configuration_files', Mock()), \ - patch.object(Postgresql, 'restore_configuration_files', Mock()), \ - patch.object(Postgresql, 'write_recovery_conf', Mock()): - with self.assertRaises(Exception) as e: - self.p.bootstrap(config) - self.assertEqual(str(e.exception), '42') + self.p.bootstrap(config) + + task = CriticalTask() + with patch.object(Postgresql, 'create_or_update_role', Mock(side_effect=Exception)): + self.p.post_bootstrap({}, task) + self.assertFalse(task.result) self.p.config.pop('pg_hba') - task = CriticalTask() self.p.post_bootstrap({}, task) - self.assertFalse(task.result) + self.assertTrue(task.result) + + self.p.bootstrap(config) + self.p.set_state('stopped') + self.p.reload_config({'authentication': {'superuser': {'username': 'p', 'password': 'p'}, + 'replication': {'username': 'r', 'password': 'r'}}, + 'listen': '*', 'retry_timeout': 10, 'parameters': {'hba_file': 'foo'}}) + with patch.object(Postgresql, 'restart', Mock()) as mock_restart: + self.p.post_bootstrap({}, task) + mock_restart.assert_called_once() def test_run_bootstrap_post_init(self): with patch('subprocess.call', Mock(return_value=1)): From e2feac87bcb8bf0c4ef64e44024a32301c907a3c Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 24 Jul 2017 14:19:19 +0200 Subject: [PATCH 4/4] Block callbacks during bootstrap (#483) It wasn't a big issue when on_start was called during normal boostrap with initdb, because usually such process is very fast. But situation is changing when we run custom bootstrap, becuase it might be a long time between cluster become connectable and end of recovery and promote. Actually situation was even worse than that, on_start was called with the `replica` argument and later on_role_changes was never called, because promote wasn't performed by Patroni. As a solution for this problem we will block any callbacks during bootstrap and explicitly call on_start after leader lock was taken. --- patroni/ha.py | 11 ++++++----- patroni/postgresql.py | 3 +++ tests/test_postgresql.py | 3 ++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/patroni/ha.py b/patroni/ha.py index a14913ce..999589a1 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -102,7 +102,6 @@ class Ha(object): self.cluster = None self.old_cluster = None self.recovering = False - self._bootstrapping = False self._post_bootstrap_task = None self._start_timeout = None self._async_executor = AsyncExecutor(self.wakeup) @@ -206,7 +205,7 @@ class Ha(object): # no initialize key and node is allowed to be master and has 'bootstrap' section in a configuration file elif self.cluster.initialize is None and not self.patroni.nofailover and 'bootstrap' in self.patroni.config: if self.dcs.initialize(create_new=True): # race for initialization - self._bootstrapping = True + self.state_handler.bootstrapping = True self._post_bootstrap_task = CriticalTask() self._async_executor.schedule('bootstrap') self._async_executor.run_async(self.state_handler.bootstrap, args=(self.patroni.config['bootstrap'],)) @@ -890,7 +889,7 @@ class Ha(object): try: if self.has_lock() and self.update_lock(): return 'updated leader lock during ' + self._async_executor.scheduled_action - elif not self._bootstrapping: + elif not self.state_handler.bootstrapping: # Don't have lock, make sure we are not starting up a master in the background if self.state_handler.role == 'master': logger.info("Demoting master during " + self._async_executor.scheduled_action) @@ -946,14 +945,16 @@ class Ha(object): if not self.state_handler.is_leader(): return 'waiting for end of recovery after bootstrap' + self.state_handler.set_role('master') self._async_executor.schedule('post_bootstrap') self._async_executor.run_async(self.state_handler.post_bootstrap, args=(self.patroni.config['bootstrap'], self._post_bootstrap_task)) return 'running post_bootstrap' - self._bootstrapping = False + self.state_handler.bootstrapping = False self.dcs.set_config_value(json.dumps(self.patroni.config.dynamic_configuration, separators=(',', ':'))) self.dcs.take_leader() + self.state_handler.call_nowait(ACTION_ON_START) self.load_cluster_from_dcs() return 'initialized a new cluster' @@ -1030,7 +1031,7 @@ class Ha(object): return msg # we've got here, so any async action has finished. - if self._bootstrapping: + if self.state_handler.bootstrapping: return self.post_bootstrap() if self.recovering and not self.state_handler.need_rewind: diff --git a/patroni/postgresql.py b/patroni/postgresql.py index 7b37b39c..61868d7d 100644 --- a/patroni/postgresql.py +++ b/patroni/postgresql.py @@ -104,6 +104,7 @@ class Postgresql(object): self._data_dir = config['data_dir'] self._config_dir = os.path.abspath(config.get('config_dir') or self._data_dir) self._pending_restart = False + self.bootstrapping = False self._running_custom_bootstrap = False self.__thread_ident = current_thread().ident @@ -742,6 +743,8 @@ class Postgresql(object): def call_nowait(self, cb_name): """ pick a callback command and call it without waiting for it to finish """ + if self.bootstrapping: + return if cb_name in (ACTION_ON_START, ACTION_ON_STOP, ACTION_ON_RESTART, ACTION_ON_ROLE_CHANGE): self.__cb_called = True diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py index 4b1342df..5e613cb6 100644 --- a/tests/test_postgresql.py +++ b/tests/test_postgresql.py @@ -487,10 +487,11 @@ class TestPostgresql(unittest.TestCase): self.assertFalse(self.p.is_running()) @patch('shlex.split', Mock(side_effect=OSError)) - @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True)) def test_call_nowait(self): self.p.set_role('replica') self.assertIsNone(self.p.call_nowait('on_start')) + self.p.bootstrapping = True + self.assertIsNone(self.p.call_nowait('on_start')) def test_non_existing_callback(self): self.assertFalse(self.p.call_nowait('foobar'))