diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst index 3a13194e..f94ac166 100644 --- a/docs/ENVIRONMENT.rst +++ b/docs/ENVIRONMENT.rst @@ -38,6 +38,7 @@ PostgreSQL - **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node. - **PATRONI\_POSTGRESQL\_CONNECT\_ADDRESS**: IP address + port through which Postgres is accessible from other nodes and applications. - **PATRONI\_POSTGRESQL\_DATA\_DIR**: The location of the Postgres data directory, either existing or to be initialized by Patroni. +- **PATRONI\_POSTGRESQL\_BIN_DIR**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The default value is an empty string meaning that PATH environment variable will be used to find the executables. - **PATRONI\_POSTGRESQL\_PGPASS**: path to the `.pgpass `__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni. - **PATRONI\_REPLICATION\_USERNAME**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication - **PATRONI\_REPLICATION\_PASSWORD**: replication password; the user will be created during initialization. diff --git a/docs/SETTINGS.rst b/docs/SETTINGS.rst index 4728dce8..24b01dd3 100644 --- a/docs/SETTINGS.rst +++ b/docs/SETTINGS.rst @@ -65,6 +65,7 @@ PostgreSQL - **connect\_address**: IP address + port through which Postgres is accessible from other nodes and applications. - **create\_replica\_methods**: an ordered list of the create methods for turning a Patroni node into a new replica. "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its own config item. - **data\_dir**: The location of the Postgres data directory, either existing or to be initialized by Patroni. +- **bin\_dir**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The default value is an empty string meaning that PATH environment variable will be used to find the executables. - **listen**: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node. - **pgpass**: path to the `.pgpass `__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni. - **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower. diff --git a/patroni/config.py b/patroni/config.py index f09cfca4..2268a59d 100644 --- a/patroni/config.py +++ b/patroni/config.py @@ -42,6 +42,7 @@ class Config(object): 'ttl': 30, 'loop_wait': 10, 'retry_timeout': 10, 'maximum_lag_on_failover': 1048576, 'postgresql': { + 'bin_dir': '', 'use_slots': True, 'parameters': {p: v[0] for p, v in Postgresql.CMDLINE_OPTIONS.items()} } @@ -193,7 +194,7 @@ class Config(object): ret[section][param] = value _set_section_values('restapi', ['listen', 'connect_address', 'certfile', 'keyfile']) - _set_section_values('postgresql', ['listen', 'connect_address', 'data_dir', 'pgpass']) + _set_section_values('postgresql', ['listen', 'connect_address', 'data_dir', 'pgpass', 'bin_dir']) def _get_auth(name): ret = {} diff --git a/patroni/ha.py b/patroni/ha.py index 8c044ba8..3367572f 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -9,6 +9,7 @@ import pytz from multiprocessing.pool import ThreadPool from patroni.async_executor import AsyncExecutor from patroni.exceptions import DCSError, PostgresConnectionException +from patroni.postgresql import ACTION_ON_START from patroni.utils import sleep logger = logging.getLogger(__name__) @@ -587,6 +588,8 @@ class Ha(object): # stops PostgreSQL, therefore, we only reload replication slots if no # asynchronous processes are running (should be always the case for the master) if not self._async_executor.busy: + if not self.state_handler.cb_called: + self.state_handler.call_nowait(ACTION_ON_START) self.state_handler.sync_replication_slots(self.cluster) except DCSError: logger.error('Error communicating with DCS') diff --git a/patroni/postgresql.py b/patroni/postgresql.py index e04b8f45..31f45148 100644 --- a/patroni/postgresql.py +++ b/patroni/postgresql.py @@ -58,6 +58,7 @@ class Postgresql(object): self.config = config self.name = config['name'] self.scope = config['scope'] + self._bin_dir = config.get('bin_dir') or '' self._database = config.get('database', 'postgres') self._data_dir = config['data_dir'] self._pending_restart = False @@ -76,6 +77,7 @@ class Postgresql(object): self._pgpass = config.get('pgpass') or os.path.join(os.path.expanduser('~'), 'pgpass') self.callback = config.get('callbacks') or {} + self.__cb_called = False config_base_name = config.get('config_base_name', 'postgresql') self._postgresql_conf = os.path.join(self._data_dir, config_base_name + '.conf') self._postgresql_base_conf_name = config_base_name + '.base.conf' @@ -131,12 +133,16 @@ class Postgresql(object): self.connection_string = 'postgres://{0}/{1}'.format( self._connect_address or self._local_address['host'] + ':' + self._local_address['port'], self._database) + def _pgcommand(self, cmd): + """Returns path to the specified PostgreSQL command""" + return os.path.join(self._bin_dir, cmd) + def pg_ctl(self, cmd, *args, **kwargs): """Builds and executes pg_ctl command :returns: `!True` when return_code == 0, otherwise `!False`""" - pg_ctl = ['pg_ctl', cmd] + pg_ctl = [self._pgcommand('pg_ctl'), cmd] if cmd in ('start', 'stop', 'restart'): pg_ctl += ['-w'] timeout = self.config.get('pg_ctl_timeout') @@ -221,7 +227,7 @@ class Postgresql(object): if not (self.config.get('use_pg_rewind') and all(self._superuser.get(n) for n in ('username', 'password'))): return False - cmd = ['pg_rewind', '--help'] + cmd = [self._pgcommand('pg_rewind'), '--help'] try: ret = subprocess.call(cmd, stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT) if ret != 0: # pg_rewind is not there, close up the shop and go home @@ -459,8 +465,15 @@ class Postgresql(object): except Exception: return False + @property + def cb_called(self): + return self.__cb_called + def call_nowait(self, cb_name): """ pick a callback command and call it without waiting for it to finish """ + if cb_name in (ACTION_ON_START, ACTION_ON_STOP, ACTION_ON_RESTART, ACTION_ON_ROLE_CHANGE): + self.__cb_called = True + if not self.callback or cb_name not in self.callback: return False cmd = self.callback[cb_name] @@ -649,7 +662,10 @@ class Postgresql(object): dsn = 'user={user} host={host} port={port} dbname={database} sslmode=prefer sslcompression=1'.format(**r) logger.info('running pg_rewind from %s', dsn) try: - return subprocess.call(['pg_rewind', '-D', self._data_dir, '--source-server', dsn], env=env) == 0 + return subprocess.call([self._pgcommand('pg_rewind'), + '-D', self._data_dir, + '--source-server', dsn, + ], env=env) == 0 except OSError: return False @@ -659,7 +675,7 @@ class Postgresql(object): # Don't try to call pg_controldata during backup restore if self._version_file_exists() and self.state != 'creating replica': try: - data = subprocess.check_output(['pg_controldata', self._data_dir]) + data = subprocess.check_output([self._pgcommand('pg_controldata'), self._data_dir]) if data: data = data.decode('utf-8').splitlines() result = {l.split(':')[0].replace('Current ', '', 1): l.split(':')[1].strip() for l in data if l} @@ -685,7 +701,7 @@ class Postgresql(object): def single_user_mode(self, command=None, options=None): """ run a given command in a single-user mode. If the command is empty - then just start and stop """ - cmd = ['postgres', '--single', '-D', self._data_dir] + cmd = [self._pgcommand('postgres'), '--single', '-D', self._data_dir] for opt, val in sorted((options or {}).items()): cmd.extend(['-c', '{0}={1}'.format(opt, val)]) # need a database name to connect @@ -788,7 +804,7 @@ class Postgresql(object): self._need_rewind = False else: self.write_recovery_conf(primary_conninfo) - ret = self.restart() + ret = self.start() if recovery else self.restart() self.set_role('replica') if change_role: @@ -957,7 +973,7 @@ $$""".format(name, ' '.join(options)), name, password, password) ret = 1 for bbfailures in range(0, maxfailures): try: - ret = subprocess.call(['pg_basebackup', '--pgdata=' + self._data_dir, + ret = subprocess.call([self._pgcommand('pg_basebackup'), '--pgdata=' + self._data_dir, '--xlog-method=stream', "--dbname=" + conn_url], env=env) if ret == 0: break diff --git a/postgres0.yml b/postgres0.yml index cb1000c5..7933eb99 100644 --- a/postgres0.yml +++ b/postgres0.yml @@ -4,10 +4,12 @@ name: postgresql0 restapi: listen: 127.0.0.1:8008 + connect_address: 127.0.0.1:8008 +# certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem +# keyfile: /etc/ssl/private/ssl-cert-snakeoil.key # authentication: # username: username # password: password - connect_address: 127.0.0.1:8008 etcd: host: 127.0.0.1:4001 @@ -30,11 +32,11 @@ bootstrap: # max_wal_senders: 5 # max_replication_slots: 5 # wal_log_hints: "on" - archive_mode: "on" - archive_timeout: 1800s - archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f - recovery_conf: - restore_command: cp ../wal_archive/%f %p +# archive_mode: "on" +# archive_timeout: 1800s +# archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f +# recovery_conf: +# restore_command: cp ../wal_archive/%f %p # some desired options for 'initdb' initdb: # Note: It needs to be a list (some options need values, others are switches) @@ -58,6 +60,7 @@ postgresql: listen: 127.0.0.1:5432 connect_address: 127.0.0.1:5432 data_dir: data/postgresql0 +# bin_dir: pgpass: /tmp/pgpass0 authentication: replication: diff --git a/postgres1.yml b/postgres1.yml index 5d2b58ee..35187a2b 100644 --- a/postgres1.yml +++ b/postgres1.yml @@ -1,105 +1,77 @@ -ttl: &ttl 30 -loop_wait: &loop_wait 10 -scope: &scope batman +scope: batman +#namespace: /service/ +name: postgresql1 + restapi: listen: 127.0.0.1:8009 connect_address: 127.0.0.1:8009 -# auth: 'username:password' # certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem # keyfile: /etc/ssl/private/ssl-cert-snakeoil.key +# authentication: +# username: username +# password: password + etcd: - scope: *scope - ttl: *ttl host: 127.0.0.1:4001 - #discovery_srv: my-etcd.domain -#consul: -# scope: *scope -# ttl: *ttl -# host: 127.0.0.1:8500 -#zookeeper: -# scope: *scope -# session_timeout: *ttl -# reconnect_timeout: *loop_wait -# hosts: -# - 127.0.0.1:2181 -# - 127.0.0.2:2181 -#exhibitor: -# poll_interval: 300 -# port: 8181 -# hosts: -# - host1 -# - host2 -# - host3 + +bootstrap: + # this section will be written into Etcd:///config after initializing new cluster + # and all other cluster members will use it as a `global configuration` + dcs: + ttl: 30 + loop_wait: 10 + retry_timeout: 10 + maximum_lag_on_failover: 1048576 + postgresql: + use_pg_rewind: true +# use_slots: true + parameters: +# wal_level: hot_standby +# hot_standby: "on" +# wal_keep_segments: 8 +# max_wal_senders: 5 +# max_replication_slots: 5 +# wal_log_hints: "on" +# archive_mode: "on" +# archive_timeout: 1800s +# archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f +# recovery_conf: +# restore_command: cp ../wal_archive/%f %p + + # some desired options for 'initdb' + initdb: # Note: It needs to be a list (some options need values, others are switches) + - encoding: UTF8 + - data-checksums + + pg_hba: # Add following lines to pg_hba.conf after running 'initdb' + - host replication replicator 127.0.0.1/32 md5 + - host all all 0.0.0.0/0 md5 +# - hostssl all all 0.0.0.0/0 md5 + + # Some additional users users which needs to be created after initializing new cluster + users: + admin: + password: admin + options: + - createrole + - createdb + postgresql: - name: postgresql1 - scope: *scope listen: 127.0.0.1:5433 connect_address: 127.0.0.1:5433 data_dir: data/postgresql1 - maximum_lag_on_failover: 1048576 # 1 megabyte in bytes - use_slots: True +# bin_dir: pgpass: /tmp/pgpass1 - initdb: ## We allow the following options to be passed on to initdb - # - auth: authmethod - # - auth-host: authmethod - # - auth-local: authmethod - - encoding: UTF8 - # - data-checksums # When pg_rewind is needed on 9.3, this needs to be enabled - # - locale: locale - # - lc-collate: locale - # - lc-ctype: locale - # - lc-messages: locale - # - lc-monetary: locale - # - lc-numeric: locale - # - lc-time: locale - # - text-search-config: CFG - # - xlogdir: directory - # - debug - # - noclean - pg_rewind: - username: postgres - password: zalando - pg_hba: - - host replication replicator 127.0.0.1/32 md5 - - host all all 0.0.0.0/0 md5 - # - hostssl all all 0.0.0.0/0 md5 - replication: - username: replicator - password: rep-pass - superuser: - username: postgres - password: zalando - admin: - username: admin - password: admin -# commented-out example for wal-e provisioning - create_replica_method: - - basebackup -# - wal_e -# commented-out example for wal-e provisioning - #wal_e: - #command: /patroni/scripts/wale_restore.py - #env_dir: /home/postgres/etc/wal-e.d/env - #threshold_megabytes: 10240 - #threshold_backup_size_percentage: 30 - #retries: 2 - #use_iam: 1 - #recovery_conf: - #restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1 - recovery_conf: - restore_command: cp ../wal_archive/%f %p + authentication: + replication: + username: replicator + password: rep-pass + superuser: + username: postgres + password: zalando parameters: - archive_mode: "on" - wal_level: hot_standby - archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f - max_wal_senders: 10 - wal_keep_segments: 8 - archive_timeout: 1800s - max_replication_slots: 10 - hot_standby: "on" - wal_log_hints: "on" unix_socket_directories: '.' tags: - nofailover: False - noloadbalance: False - clonefrom: False + nofailover: false + noloadbalance: false + clonefrom: false diff --git a/postgres2.yml b/postgres2.yml index 48c3b55d..3ea34b9d 100644 --- a/postgres2.yml +++ b/postgres2.yml @@ -1,106 +1,78 @@ -ttl: &ttl 30 -loop_wait: &loop_wait 10 -scope: &scope batman +scope: batman +#namespace: /service/ +name: postgresql2 + restapi: listen: 127.0.0.1:8010 connect_address: 127.0.0.1:8010 - auth: 'username:password' # certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem # keyfile: /etc/ssl/private/ssl-cert-snakeoil.key + authentication: + username: username + password: password + etcd: - scope: *scope - ttl: *ttl host: 127.0.0.1:4001 - #discovery_srv: my-etcd.domain -#consul: -# scope: *scope -# ttl: *ttl -# host: 127.0.0.1:8500 -#zookeeper: -# scope: *scope -# session_timeout: *ttl -# reconnect_timeout: *loop_wait -# hosts: -# - 127.0.0.1:2181 -# - 127.0.0.2:2181 -#exhibitor: -# poll_interval: 300 -# port: 8181 -# hosts: -# - host1 -# - host2 -# - host3 + +bootstrap: + # this section will be written into Etcd:///config after initializing new cluster + # and all other cluster members will use it as a `global configuration` + dcs: + ttl: 30 + loop_wait: 10 + retry_timeout: 10 + maximum_lag_on_failover: 1048576 + postgresql: + use_pg_rewind: true +# use_slots: true + parameters: +# wal_level: hot_standby +# hot_standby: "on" +# wal_keep_segments: 8 +# max_wal_senders: 5 +# max_replication_slots: 5 +# wal_log_hints: "on" +# archive_mode: "on" +# archive_timeout: 1800s +# archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f +# recovery_conf: +# restore_command: cp ../wal_archive/%f %p + + # some desired options for 'initdb' + initdb: # Note: It needs to be a list (some options need values, others are switches) + - encoding: UTF8 + - data-checksums + + pg_hba: # Add following lines to pg_hba.conf after running 'initdb' + - host replication replicator 127.0.0.1/32 md5 + - host all all 0.0.0.0/0 md5 +# - hostssl all all 0.0.0.0/0 md5 + + # Some additional users users which needs to be created after initializing new cluster + users: + admin: + password: admin + options: + - createrole + - createdb + postgresql: - name: postgresql2 - scope: *scope listen: 127.0.0.1:5434 connect_address: 127.0.0.1:5434 data_dir: data/postgresql2 - maximum_lag_on_failover: 1048576 # 1 megabyte in bytes - use_slots: True +# bin_dir: pgpass: /tmp/pgpass2 - initdb: ## We allow the following options to be passed on to initdb - # - auth: authmethod - # - auth-host: authmethod - # - auth-local: authmethod - - encoding: UTF8 - # - data-checksums # When pg_rewind is needed on 9.3, this needs to be enabled - # - locale: locale - # - lc-collate: locale - # - lc-ctype: locale - # - lc-messages: locale - # - lc-monetary: locale - # - lc-numeric: locale - # - lc-time: locale - # - text-search-config: CFG - # - xlogdir: directory - # - debug - # - noclean - pg_rewind: - username: postgres - password: zalando - pg_hba: - - host replication replicator 127.0.0.1/32 md5 - - host all all 0.0.0.0/0 md5 - # - hostssl all all 0.0.0.0/0 md5 - replication: - username: replicator - password: rep-pass - superuser: - username: postgres - password: zalando - admin: - username: admin - password: admin -# commented-out example for wal-e provisioning - create_replica_method: - - basebackup -# - wal_e -# commented-out example for wal-e provisioning - #wal_e: - #command: /patroni/scripts/wale_restore.py - #env_dir: /home/postgres/etc/wal-e.d/env - #threshold_megabytes: 10240 - #threshold_backup_size_percentage: 30 - #retries: 2 - #use_iam: 1 - #recovery_conf: - #restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1 - recovery_conf: - restore_command: cp ../wal_archive/%f %p + authentication: + replication: + username: replicator + password: rep-pass + superuser: + username: postgres + password: zalando parameters: - archive_mode: "on" - wal_level: hot_standby - archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f - max_wal_senders: 10 - wal_keep_segments: 8 - archive_timeout: 1800s - max_replication_slots: 10 - hot_standby: "on" - wal_log_hints: "on" unix_socket_directories: '.' tags: - nofailover: False - noloadbalance: False - clonefrom: False - replicatefrom: postgresql1 + nofailover: false + noloadbalance: false + clonefrom: false + replicatefrom: postgres1 diff --git a/tests/test_api.py b/tests/test_api.py index ea68486c..3e90346a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -179,7 +179,6 @@ class TestRestApiHandler(unittest.TestCase): MockRestApiServer(RestApiHandler, 'POST /reload HTTP/1.0' + self._authorization) self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /reload HTTP/1.0' + self._authorization)) - #@patch.object(MockPatroni, 'dcs') def test_do_POST_restart(self): request = 'POST /restart HTTP/1.0' + self._authorization self.assertIsNotNone(MockRestApiServer(RestApiHandler, request)) @@ -189,7 +188,8 @@ class TestRestApiHandler(unittest.TestCase): post = request + '\nContent-Length: ' - def make_request(request): + def make_request(request=None, **kwargs): + request = json.dumps(kwargs) if request is None else request return '{0}{1}\n\n{2}'.format(post, len(request), request) # empty request @@ -199,29 +199,28 @@ class TestRestApiHandler(unittest.TestCase): request = make_request('foobar=baz') MockRestApiServer(RestApiHandler, request) # wrong role - request = make_request('{"schedule": "2016-08-20 12:45TZ+1", "role": "unknown", "postgres_version": "9.5.3"}') + request = make_request(schedule=future_restart_time.isoformat(), role='unknown', postgres_version='9.5.3') MockRestApiServer(RestApiHandler, request) # wrong version - request = make_request('{"schedule": "2016-08-20 12:45TZ+1", "role": "master", "postgres_version": "9.5.3.1"}') + request = make_request(schedule=future_restart_time.isoformat(), role='master', postgres_version='9.5.3.1') MockRestApiServer(RestApiHandler, request) # unknown filter - request = make_request('{"schedule": "2016-08-29 12:45TZ+1", "batman": "lives"}') + request = make_request(schedule=future_restart_time.isoformat(), batman='lives') MockRestApiServer(RestApiHandler, request) # incorrect schedule - request = make_request('{"schedule": "2016-08-42 12:45TZ+1", "role": "master"}') + request = make_request(schedule='2016-08-42 12:45TZ+1', role='master') MockRestApiServer(RestApiHandler, request) # everything fine, but the schedule is missing - request = make_request('{"role": "master", "postgres_version": "9.5.2"}') + request = make_request(role='master', postgres_version='9.5.2') MockRestApiServer(RestApiHandler, request) for retval in (True, False): with patch.object(MockHa, 'schedule_future_restart', Mock(return_value=retval)): - request = make_request('{"schedule": "2016-08-29 12:45TZ+1"}') + request = make_request(schedule=future_restart_time.isoformat()) MockRestApiServer(RestApiHandler, request) with patch.object(MockHa, 'restart', Mock(return_value=(retval, "foo"))): - request = make_request('{"role": "master", "postgres_version": "9.5.2"}') + request = make_request(role='master', postgres_version='9.5.2') MockRestApiServer(RestApiHandler, request) - #@patch.object(MockPatroni, 'dcs') def test_do_DELETE_restart(self): for retval in (True, False): with patch.object(MockHa, 'delete_future_restart', Mock(return_value=retval)): diff --git a/tests/test_ctl.py b/tests/test_ctl.py index 0bf6eaf8..d4e6db66 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -29,9 +29,9 @@ def test_rw_config(): os.rmdir(CONFIG_FILE_PATH) -@patch('patroni.ctl.load_config', Mock(return_value={'postgresql': {'data_dir': '.', 'parameters': {}, 'retry_timeout': 5}, - 'restapi': {'auth': 'u:p', 'listen': ''}, - 'etcd': {'host': 'localhost:4001'}})) +@patch('patroni.ctl.load_config', + Mock(return_value={'postgresql': {'data_dir': '.', 'parameters': {}, 'retry_timeout': 5}, + 'restapi': {'auth': 'u:p', 'listen': ''}, 'etcd': {'host': 'localhost:4001'}})) class TestCtl(unittest.TestCase): @patch('socket.getaddrinfo', socket_getaddrinfo)