mirror of
https://github.com/outbackdingo/patroni.git
synced 2026-08-29 00:49:32 +00:00
Compare commits
2
Commits
v0.80
..
feature/bdr
| Author | SHA1 | Date | |
|---|---|---|---|
 Oleksii Kliukin
|
1eeb544431 | ||
|
Oleksii Kliukin
|
494565e6bb |
+5
-23
@@ -1,32 +1,14 @@
|
||||
sudo: required
|
||||
language: python
|
||||
addons:
|
||||
postgresql: "9.5"
|
||||
env:
|
||||
global:
|
||||
- BOTO_CONFIG='' ETCDVERSION=2.2.5
|
||||
matrix:
|
||||
- TEST_SUITE="python setup.py test"
|
||||
- TEST_SUITE="behave"
|
||||
python:
|
||||
- "2.7"
|
||||
- "3.3"
|
||||
- "3.4"
|
||||
- "3.5"
|
||||
install:
|
||||
- sudo /etc/init.d/postgresql stop
|
||||
- sudo apt-get -y remove --purge postgresql-9.1 postgresql-9.2 postgresql-9.3 postgresql-9.4
|
||||
- sudo apt-get -y autoremove
|
||||
- sudo apt-key adv --keyserver keys.gnupg.net --recv-keys 7FCC7D46ACCC4CF8
|
||||
- sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main 9.5" >> /etc/apt/sources.list.d/postgresql.list'
|
||||
- sudo apt-get update
|
||||
- sudo apt-get -y install postgresql-9.5
|
||||
- sudo /etc/init.d/postgresql stop
|
||||
- pip install -r requirements.txt
|
||||
- curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd
|
||||
- pip install behave codacy-coverage coverage coveralls
|
||||
- if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt --use-mirrors; fi
|
||||
- if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt --use-mirrors; fi
|
||||
- pip install coveralls
|
||||
script:
|
||||
- PATH=.:$PATH $TEST_SUITE
|
||||
- python setup.py test
|
||||
- python setup.py flake8
|
||||
after_success:
|
||||
- coveralls
|
||||
- if [[ -f coverage.xml ]]; then python-codacy-coverage -r coverage.xml; fi
|
||||
|
||||
-12
@@ -1,12 +0,0 @@
|
||||
approvals:
|
||||
# PR needs at least 4 approvals
|
||||
minimum: 1
|
||||
# approval = comment that matches this regex
|
||||
pattern: "^:?\\+1:?$"
|
||||
from:
|
||||
# commenter must be either one of:
|
||||
# a public zalando org member
|
||||
orgs:
|
||||
- zalando
|
||||
# a collaborator of the repo
|
||||
collaborators: true
|
||||
+13
-11
@@ -6,28 +6,30 @@ MAINTAINER Feike Steenbergen <[email protected]>
|
||||
# We need curl
|
||||
RUN apt-get update -y && apt-get install curl -y
|
||||
|
||||
# Add PGDG repositories
|
||||
# Add PGDG and BDR repositories
|
||||
RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
|
||||
RUN echo "deb http://packages.2ndquadrant.com/bdr/apt/ $(lsb_release -cs)-2ndquadrant main" >> /etc/apt/sources.list.d/pgdg.list
|
||||
RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
|
||||
# import the BDR key
|
||||
RUN curl -s -o - http://packages.2ndquadrant.com/bdr/apt/AA7A6805.asc | sudo apt-key add -
|
||||
|
||||
RUN apt-get update -y
|
||||
RUN apt-get upgrade -y
|
||||
|
||||
ENV PGVERSION 9.5
|
||||
RUN apt-get install postgresql-${PGVERSION} postgresql-server-dev-${PGVERSION} -y
|
||||
RUN apt-get install python python-dev python-pip -y
|
||||
ADD requirements-py2.txt /requirements-py2.txt
|
||||
RUN pip install -r /requirements-py2.txt
|
||||
ENV PGVERSION 9.4
|
||||
ENV PGTYPE postgresql-bdr
|
||||
ENV PGCOMPATIBLETYPE postgresql
|
||||
|
||||
RUN apt-get install python python-yaml python-requests python-boto ${PGTYPE}-${PGVERSION} ${PGTYPE}-${PGVERSION}-bdr-plugin python-dnspython python-kazoo python-pip -y
|
||||
RUN apt-get install python-dev ${PGTYPE}-server-dev-${PGVERSION} -y
|
||||
RUN pip install python-etcd psycopg2
|
||||
|
||||
ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH
|
||||
|
||||
ADD patroni.py /patroni.py
|
||||
ADD patronictl.py /patronictl.py
|
||||
ADD patroni/ /patroni
|
||||
|
||||
RUN ln -s /patroni.py /usr/local/bin/patroni
|
||||
RUN ln -s /patronictl.py /usr/local/bin/patronictl
|
||||
|
||||
ENV ETCDVERSION 2.2.5
|
||||
ENV ETCDVERSION 2.0.13
|
||||
RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C /bin --strip=1 --wildcards --no-anchored etcd etcdctl
|
||||
|
||||
### Setting up a simple script that will serve as an entrypoint
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
Alexander Kukushkin <[email protected]>
|
||||
Feike Steenbergen <[email protected]>
|
||||
Oleksii Kliukin <[email protected]>
|
||||
+30
-2
@@ -75,10 +75,13 @@ For an example file, see ``postgres0.yml``. Regarding settings:
|
||||
- *port*: Exhibitor port.
|
||||
- *hosts*: initial list of Exhibitor (ZooKeeper) nodes in format: ['host1', 'host2', 'etc...' ]. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.
|
||||
|
||||
- *bdr*:
|
||||
- *enable*: on if you want to enable BDR
|
||||
- *database*: database name to support BDR (only a single database is supported)
|
||||
|
||||
- *postgresql*:
|
||||
- *name*: the name of the Postgres host. Must be unique for the cluster.
|
||||
- *listen*: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. The first address from this list will be used by Patroni to establish local connections to the PostgreSQL node.
|
||||
|
||||
- *listen*: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication.
|
||||
- *connect\_address*: IP address + port through which Postgres is accessible from other nodes and applications.
|
||||
- *data\_dir*: file path to initialize and store Postgres data files.
|
||||
- *maximum\_lag\_on\_failover*: the maximum bytes a follower may lag.
|
||||
@@ -164,6 +167,31 @@ Choosing your replication schema is dependent on your business
|
||||
considerations. Investigate both async and sync replication, as well as other
|
||||
HA solutions, to determine which solution is best for you.
|
||||
|
||||
You can also use BDR (bi-directional replication) if you have a compatible
|
||||
PostgreSQL version with the BDR plugin installed (see http://bdr-project.org/docs/next/installation.html).
|
||||
It will require adding a BDR shared_library to your configuration, as well as
|
||||
setting the following options for postgresql (see http://bdr-project.org/docs/next/settings-prerequisite.html):
|
||||
|
||||
.. code:: YAML
|
||||
|
||||
max_worker_processes: 10
|
||||
max_replication_slots: 10
|
||||
max_wal_senders: 10
|
||||
shared_preload_libraries: 'bdr'
|
||||
track_commit_timestamp: 'on'
|
||||
wal_level: 'logical'
|
||||
|
||||
At the moment Patroni BDR is not compatible with a streaming replication,
|
||||
if BDR is enabled normal replica node won't be able to join. This is not
|
||||
a principal limitation of BDR, and we might resolve this in the future
|
||||
(although 'promotion' will only work between nodes running a physical
|
||||
replication, i.e. a replica won't be able to attach to a different
|
||||
multimaster node).
|
||||
|
||||
Another limitation is that only one database is supported at the moment.
|
||||
BDR requires the replication user to be also a superuser, so you might
|
||||
want to excersie extra caution when choosing the password for this user.
|
||||
|
||||
Applications Should Not Use Superusers
|
||||
--------------------------------------
|
||||
|
||||
|
||||
@@ -1,16 +1,4 @@
|
||||
Failover
|
||||
========
|
||||
- When determining who should become master, include the minor version of PostgreSQL in the decision.
|
||||
- Create a way to disable governance of a cluster, something like the existence of a "nogover" or "admin" file in PGDATA will stop patroni from changing the cluster state.
|
||||
|
||||
Configuration
|
||||
==============
|
||||
- Provide a way to change postgresql.conf and pg_hba.conf of a running cluster on the Patroni level, without changing individual nodes.
|
||||
- Provide hooks to store and retrieve cluster-wide passwords without exposing them in a plain-text form to unauthorized users.
|
||||
- Implement patronictl command to create initial configuration of the cluster with leader and member keys fixed to the user-supplied values in order to simplify migrations.
|
||||
- Implement support for consul in addtion to etcd and zookeeper
|
||||
- Complete zookeeper support in patronictl
|
||||
|
||||
Documentation
|
||||
==============
|
||||
- Document how to run cascading replication and possibly initialize the cluster without an access to the master node.
|
||||
- When determining who should become master, include the minor version of PostgreSQL in the decision
|
||||
- Create a way to disable governance of a cluster, something like the existence of a "nogover" or "admin" file in PGDATA will stop governor from changing the cluster state
|
||||
|
||||
+14
-14
@@ -79,12 +79,7 @@ then
|
||||
ETCD_CLUSTER="127.0.0.1:4001"
|
||||
fi
|
||||
|
||||
mkdir -p ~postgres/.config/patroni
|
||||
cat > ~postgres/.config/patroni/patronictl.yaml <<__EOF__
|
||||
{dcs_api: 'etcd://${ETCD_CLUSTER}', namespace: /service/}
|
||||
__EOF__
|
||||
|
||||
cat > /patroni/postgres.yaml <<__EOF__
|
||||
cat > /patroni/postgres.yml <<__EOF__
|
||||
|
||||
ttl: &ttl 30
|
||||
loop_wait: &loop_wait 10
|
||||
@@ -97,6 +92,9 @@ etcd:
|
||||
scope: *scope
|
||||
ttl: *ttl
|
||||
host: ${ETCD_CLUSTER}
|
||||
bdr:
|
||||
enable: 'on'
|
||||
database: 'bdrtest'
|
||||
postgresql:
|
||||
name: ${HOSTNAME}
|
||||
scope: *scope
|
||||
@@ -124,22 +122,24 @@ postgresql:
|
||||
archive_command: 'true'
|
||||
max_wal_senders: 20
|
||||
listen_addresses: 0.0.0.0
|
||||
max_wal_size: 1GB
|
||||
min_wal_size: 128MB
|
||||
checkpoint_segments: 64
|
||||
wal_keep_segments: 64
|
||||
archive_timeout: 1800s
|
||||
max_replication_slots: 20
|
||||
hot_standby: "on"
|
||||
max_worker_processes: 10
|
||||
max_replication_slots: 10
|
||||
max_wal_senders: 10
|
||||
shared_preload_libraries: 'bdr'
|
||||
track_commit_timestamp: true
|
||||
wal_level: 'logical'
|
||||
__EOF__
|
||||
|
||||
cat /patroni/postgres.yaml
|
||||
cat /patroni/postgres.yml
|
||||
|
||||
if [ ! -z $CHEAT ]
|
||||
then
|
||||
while :
|
||||
do
|
||||
sleep 60
|
||||
done
|
||||
exec bash
|
||||
else
|
||||
exec python /patroni.py /patroni/postgres.yaml
|
||||
exec python /patroni.py /patroni/postgres.yml
|
||||
fi
|
||||
|
||||
@@ -8,6 +8,3 @@ Scripts supplied:
|
||||
### patroni.upstart.conf
|
||||
|
||||
Upstart job for Ubuntu 12.04 or 14.04. Requires Upstart > 1.4. Intended for systems where Patroni has been installed on a base system, rather than in Docker.
|
||||
|
||||
### patroni.service
|
||||
Systemd service file, to be copied to /etc/systemd/system/patroni.service, tested on Centos 7.1 with Patroni installed from pip.
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
# This is an example systemd config file for Patroni
|
||||
# You can copy it to "/etc/systemd/system/patroni.service",
|
||||
|
||||
[Unit]
|
||||
Description=Runners to orchestrate a high-availability PostgreSQL
|
||||
After=syslog.target network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
|
||||
User=postgres
|
||||
Group=postgres
|
||||
|
||||
# Where to send early-startup messages from the server
|
||||
# This is normally controlled by the global default set by systemd
|
||||
# StandardOutput=syslog
|
||||
|
||||
ExecStart=/bin/patroni /etc/patroni.yml
|
||||
|
||||
# Give a reasonable amount of time for the server to start up/shut down
|
||||
TimeoutSec=10
|
||||
|
||||
# Do not restart the service if it crashes, we want to manually inspect database on failure
|
||||
Restart=no
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
Feature: basic replication
|
||||
We should check that the basic bootstrapping, replication and failover works.
|
||||
|
||||
Scenario: check replication of a single table
|
||||
Given I start postgres0
|
||||
And postgres0 is a leader after 10 seconds
|
||||
And I start postgres1
|
||||
When I add the table foo to postgres0
|
||||
Then table foo is present on postgres1 after 15 seconds
|
||||
|
||||
Scenario: check the basic failover
|
||||
When I kill postgres0
|
||||
Then postgres1 role is the primary after 30 seconds
|
||||
When I start postgres0
|
||||
Then postgres0 role is the secondary after 15 seconds
|
||||
When I add the table bar to postgres1
|
||||
Then table bar is present on postgres0 after 10 seconds
|
||||
@@ -1,13 +0,0 @@
|
||||
Feature: cascading replication
|
||||
We should check that patroni can do base backup and streaming from the replica
|
||||
|
||||
Scenario: check a base backup from the replica
|
||||
Given I start postgres0
|
||||
And postgres0 is a leader after 10 seconds
|
||||
And I start postgres1
|
||||
And replication works from postgres0 to postgres1 after 15 seconds
|
||||
And I create label with "postgres0" in postgres0 data directory
|
||||
And I create label with "postgres1" in postgres1 data directory
|
||||
And I configure and start postgres2 with a tag clonefrom postgres1
|
||||
Then replication works from postgres0 to postgres2 after 30 seconds
|
||||
And there is a label with "postgres1" in postgres2 data directory
|
||||
@@ -1,305 +0,0 @@
|
||||
import os
|
||||
import psycopg2
|
||||
import requests
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import yaml
|
||||
|
||||
|
||||
class PatroniController(object):
|
||||
PATRONI_CONFIG = '{}.yml'
|
||||
""" starts and stops individual patronis"""
|
||||
|
||||
def __init__(self):
|
||||
self._output_dir = None
|
||||
self._patroni_path = None
|
||||
self._connections = {}
|
||||
self._config = {}
|
||||
self._connstring = {}
|
||||
self._cursors = {}
|
||||
self._log = {}
|
||||
self._processes = {}
|
||||
|
||||
@property
|
||||
def patroni_path(self):
|
||||
if self._patroni_path is None:
|
||||
cwd = os.path.realpath(__file__)
|
||||
while True:
|
||||
path, entry = os.path.split(cwd)
|
||||
cwd = path
|
||||
if entry == 'features' or cwd == '/':
|
||||
break
|
||||
self._patroni_path = cwd
|
||||
return self._patroni_path
|
||||
|
||||
def data_dir(self, pg_name):
|
||||
return os.path.join(self.patroni_path, 'data', pg_name)
|
||||
|
||||
def write_label(self, pg_name, content):
|
||||
with open(os.path.join(self.data_dir(pg_name), 'label'), 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
def read_label(self, pg_name):
|
||||
content = None
|
||||
try:
|
||||
with open(os.path.join(self.data_dir(pg_name), 'label'), 'r') as f:
|
||||
content = f.read()
|
||||
except IOError:
|
||||
return None
|
||||
return content.strip()
|
||||
|
||||
def start(self, pg_name, max_wait_limit=20, tags=None):
|
||||
if not self._is_running(pg_name):
|
||||
if pg_name in self._processes:
|
||||
del self._processes[pg_name]
|
||||
cwd = self.patroni_path
|
||||
self._log[pg_name] = open(os.path.join(self._output_dir, 'patroni_{0}.log'.format(pg_name)), 'a')
|
||||
|
||||
self._config[pg_name] = self._make_patroni_test_config(pg_name, tags=tags)
|
||||
|
||||
p = subprocess.Popen(['coverage', 'run', '--branch', '--source=patroni', '-p', 'patroni.py', self._config[pg_name]],
|
||||
stdout=self._log[pg_name], stderr=subprocess.STDOUT, cwd=cwd)
|
||||
if not (p and p.pid and p.poll() is None):
|
||||
assert False, "PostgreSQL {0} is not running after being started".format(pg_name)
|
||||
self._processes[pg_name] = p
|
||||
# wait while patroni is available for queries, but not more than 10 seconds.
|
||||
for _ in range(max_wait_limit):
|
||||
if self.query(pg_name, "SELECT 1", fail_ok=True) is not None:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
assert False,\
|
||||
"Patroni instance is not available for queries after {0} seconds".format(max_wait_limit)
|
||||
|
||||
def stop(self, pg_name, kill=False, timeout=15):
|
||||
start_time = time.time()
|
||||
while self._is_running(pg_name):
|
||||
if not kill:
|
||||
self._processes[pg_name].terminate()
|
||||
else:
|
||||
self._processes[pg_name].kill()
|
||||
time.sleep(1)
|
||||
if not kill and time.time() - start_time > timeout:
|
||||
kill = True
|
||||
if self._log.get('pg_name') and not self._log['pg_name'].closed:
|
||||
self._log[pg_name].close()
|
||||
if pg_name in self._processes:
|
||||
del self._processes[pg_name]
|
||||
|
||||
def query(self, pg_name, query, fail_ok=False):
|
||||
try:
|
||||
cursor = self._cursor(pg_name)
|
||||
cursor.execute(query)
|
||||
return cursor
|
||||
except psycopg2.Error:
|
||||
if fail_ok:
|
||||
return None
|
||||
else:
|
||||
raise
|
||||
|
||||
def check_role_has_changed_to(self, pg_name, new_role, timeout=10):
|
||||
bound_time = time.time() + timeout
|
||||
recovery_status = False if new_role == 'primary' else True
|
||||
role_has_changed = False
|
||||
while not role_has_changed:
|
||||
cur = self.query(pg_name, "SELECT pg_is_in_recovery()", fail_ok=True)
|
||||
if cur:
|
||||
row = cur.fetchone()
|
||||
if row and len(row) > 0 and row[0] == recovery_status:
|
||||
role_has_changed = True
|
||||
if time.time() > bound_time:
|
||||
break
|
||||
time.sleep(1)
|
||||
return role_has_changed
|
||||
|
||||
def stop_all(self):
|
||||
for patroni in self._processes.copy():
|
||||
self.stop(patroni)
|
||||
|
||||
def create_and_set_output_directory(self, feature_name):
|
||||
feature_dir = os.path.join(self.patroni_path, "features", "output",
|
||||
feature_name.replace(' ', '_'))
|
||||
if os.path.exists(feature_dir):
|
||||
shutil.rmtree(feature_dir)
|
||||
os.makedirs(feature_dir)
|
||||
self._output_dir = feature_dir
|
||||
|
||||
def _is_running(self, pg_name):
|
||||
return pg_name in self._processes and self._processes[pg_name].pid and (self._processes[pg_name].poll() is None)
|
||||
|
||||
def _make_patroni_test_config(self, pg_name, tags=None):
|
||||
patroni_config_name = PatroniController.PATRONI_CONFIG.format(pg_name)
|
||||
patroni_config_path = os.path.join(self._output_dir, patroni_config_name)
|
||||
|
||||
with open(patroni_config_name) as f:
|
||||
config = yaml.load(f)
|
||||
postgresql = config['postgresql']
|
||||
postgresql['name'] = pg_name
|
||||
postgresql['data_dir'] = 'data/{0}'.format(pg_name)
|
||||
postgresql_params = postgresql['parameters']
|
||||
postgresql_params['logging_collector'] = 'on'
|
||||
postgresql_params['log_destination'] = 'csvlog'
|
||||
postgresql_params['log_directory'] = self._output_dir
|
||||
postgresql_params['log_filename'] = '{0}.log'.format(pg_name)
|
||||
postgresql_params['log_statement'] = 'all'
|
||||
postgresql_params['log_min_messages'] = 'debug1'
|
||||
postgresql_params['unix_socket_directories'] = '.'
|
||||
|
||||
if tags:
|
||||
config['tags'] = tags
|
||||
|
||||
with open(patroni_config_path, 'w') as f:
|
||||
yaml.dump(config, f, default_flow_style=False)
|
||||
|
||||
return patroni_config_path
|
||||
|
||||
def _make_connstring(self, pg_name):
|
||||
if pg_name in self._connstring:
|
||||
return self._connstring[pg_name]
|
||||
try:
|
||||
patroni_path = self.patroni_path
|
||||
with open(os.path.join(patroni_path, PatroniController.PATRONI_CONFIG.format(pg_name)), 'r') as f:
|
||||
config = yaml.load(f)
|
||||
except IOError:
|
||||
return None
|
||||
connstring = config['postgresql']['connect_address']
|
||||
if ':' in connstring:
|
||||
address, port = connstring.split(':')
|
||||
else:
|
||||
address = connstring
|
||||
port = '5432'
|
||||
user = "postgres"
|
||||
dbname = "postgres"
|
||||
self._connstring[pg_name] = "host={0} port={1} dbname={2} user={3}".format(address, port, dbname, user)
|
||||
return self._connstring[pg_name]
|
||||
|
||||
def _connection(self, pg_name):
|
||||
if pg_name not in self._connections or self._connections[pg_name].closed:
|
||||
conn = psycopg2.connect(self._make_connstring(pg_name))
|
||||
conn.autocommit = True
|
||||
self._connections[pg_name] = conn
|
||||
return self._connections[pg_name]
|
||||
|
||||
def _cursor(self, pg_name):
|
||||
if pg_name not in self._cursors or self._cursors[pg_name].closed:
|
||||
cursor = self._connection(pg_name).cursor()
|
||||
self._cursors[pg_name] = cursor
|
||||
return self._cursors[pg_name]
|
||||
|
||||
|
||||
class EtcdController(object):
|
||||
|
||||
""" handles all etcd related tasks, used for the tests setup and cleanup """
|
||||
ETCD_VERSION_URL = 'http://127.0.0.1:2379/version'
|
||||
ETCD_CLEANUP_URL = 'http://127.0.0.1:2379/v2/keys/service/batman?recursive=true'
|
||||
|
||||
def __init__(self, log_directory):
|
||||
self.handle = None
|
||||
self.work_directory = None
|
||||
self.log_directory = log_directory
|
||||
self.log_file = None
|
||||
self.pid = None
|
||||
self.start_timeout = 5
|
||||
|
||||
def start(self):
|
||||
""" start etcd if it's not already running """
|
||||
if self._is_running():
|
||||
return True
|
||||
self.work_directory = tempfile.mkdtemp()
|
||||
# etcd is running throughout the tests, no need to append to the log
|
||||
output_dir = os.path.join(self.log_directory, "features", "output")
|
||||
if not os.path.exists(output_dir):
|
||||
os.makedirs(output_dir)
|
||||
self.log_file = open(os.path.join(output_dir, 'etcd.log'), 'w')
|
||||
self.handle =\
|
||||
subprocess.Popen(["etcd", "--debug", "--data-dir", self.work_directory],
|
||||
stdout=self.log_file, stderr=subprocess.STDOUT)
|
||||
start_time = time.time()
|
||||
while (not self._is_running()):
|
||||
if time.time() - start_time > self.start_timeout:
|
||||
assert False, "Failed to start etcd"
|
||||
time.sleep(1)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def query(key):
|
||||
""" query etcd for a value of a given key """
|
||||
r = requests.get("http://127.0.0.1:2379/v2/keys/service/batman/{0}".format(key))
|
||||
if r.ok:
|
||||
content = r.json()
|
||||
if content:
|
||||
return content.get('node', {}).get('value')
|
||||
return None
|
||||
|
||||
def stop_and_remove_work_directory(self, timeout=15):
|
||||
""" terminate etcd and wipe out the temp work directory, but only if we actually started it"""
|
||||
kill = False
|
||||
start_time = time.time()
|
||||
while self._is_running() and self.handle:
|
||||
if not kill:
|
||||
self.handle.terminate()
|
||||
else:
|
||||
self.handle.kill()
|
||||
time.sleep(1)
|
||||
if not kill and time.time() - start_time > timeout:
|
||||
kill = True
|
||||
self.handle = None
|
||||
if self.log_file and not self.log_file.closed:
|
||||
self.log_file.close()
|
||||
if self.work_directory:
|
||||
shutil.rmtree(self.work_directory)
|
||||
self.work_directory = None
|
||||
|
||||
@staticmethod
|
||||
def cleanup_service_tree():
|
||||
""" clean all contents stored in the tree used for the tests """
|
||||
r = None
|
||||
try:
|
||||
r = requests.delete(EtcdController.ETCD_CLEANUP_URL)
|
||||
if r and not r.ok:
|
||||
assert False,\
|
||||
"request to cleanup the etcd contents was not successfull: status code {0}".format(r.status_code)
|
||||
except requests.exceptions.RequestException as e:
|
||||
assert False, "exception when cleaning up etcd contents: {0}".format(e)
|
||||
|
||||
@staticmethod
|
||||
def _is_running():
|
||||
# if etcd is running, but we didn't start it
|
||||
try:
|
||||
r = requests.get(EtcdController.ETCD_VERSION_URL)
|
||||
running = (r and r.ok and b'etcdserver' in r.content)
|
||||
except requests.ConnectionError:
|
||||
running = False
|
||||
return running
|
||||
|
||||
|
||||
# actions to execute on start/stop of the tests and before running invidual features
|
||||
def before_all(context):
|
||||
context.pctl = PatroniController()
|
||||
context.etcd_ctl = EtcdController(context.pctl.patroni_path)
|
||||
context.etcd_ctl.start()
|
||||
try:
|
||||
context.etcd_ctl.cleanup_service_tree()
|
||||
except AssertionError: # after.all handlers won't be executed in before.all
|
||||
context.etcd_ctl.stop_and_remove_work_directory()
|
||||
raise
|
||||
|
||||
|
||||
def after_all(context):
|
||||
context.etcd_ctl.stop_and_remove_work_directory()
|
||||
subprocess.call(['coverage', 'combine'])
|
||||
subprocess.call(['coverage', 'report'])
|
||||
|
||||
|
||||
def before_feature(context, feature):
|
||||
""" create per-feature output directory to collect Patroni and PostgreSQL logs """
|
||||
context.pctl.create_and_set_output_directory(feature.name)
|
||||
|
||||
|
||||
def after_feature(context, feature):
|
||||
""" stop all Patronis, remove their data directory and cleanup the keys in etcd """
|
||||
context.pctl.stop_all()
|
||||
shutil.rmtree(os.path.join(context.pctl.patroni_path, 'data'))
|
||||
context.etcd_ctl.cleanup_service_tree()
|
||||
@@ -1,48 +0,0 @@
|
||||
Feature: patroni api
|
||||
We should check that patroni correctly responds to valid and not-valid API requests.
|
||||
|
||||
Scenario: check API requests on a stand-alone server
|
||||
Given I start postgres0
|
||||
And postgres0 is a leader after 10 seconds
|
||||
When I issue a GET request to http://127.0.0.1:8008/
|
||||
Then I receive a response code 200
|
||||
And I receive a response state running
|
||||
And I receive a response role master
|
||||
When I issue a GET request to http://127.0.0.1:8008/replica
|
||||
Then I receive a response code 503
|
||||
When I issue an empty POST request to http://127.0.0.1:8008/reinitialize
|
||||
Then I receive a response code 503
|
||||
And I receive a response text "I am the leader, can not reinitialize"
|
||||
When I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0
|
||||
Then I receive a response code 500
|
||||
And I receive a response text "failover is not possible: cluster does not have members except leader"
|
||||
When I issue an empty POST request to http://127.0.0.1:8008/failover
|
||||
Then I receive a response code 400
|
||||
And I receive a response text "No values given for required parameters leader and member"
|
||||
|
||||
Scenario: check API requests for the primary-replica pair
|
||||
Given I start postgres1
|
||||
And replication works from postgres0 to postgres1 after 15 seconds
|
||||
When I issue a GET request to http://127.0.0.1:8009/replica
|
||||
Then I receive a response code 200
|
||||
And I receive a response state running
|
||||
And I receive a response role replica
|
||||
When I issue an empty POST request to http://127.0.0.1:8009/reinitialize
|
||||
Then I receive a response code 200
|
||||
Given replication works from postgres0 to postgres1 after 10 seconds
|
||||
When I issue an empty POST request to http://127.0.0.1:8008/restart
|
||||
Then I receive a response code 200
|
||||
And postgres0 is a leader after 5 seconds
|
||||
|
||||
Scenario: check the failover via the API
|
||||
Given I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0,candidate=postgres1
|
||||
Then I receive a response code 200
|
||||
And postgres1 is a leader after 5 seconds
|
||||
And replication works from postgres1 to postgres0 after 15 seconds
|
||||
|
||||
Scenario: check the scheduled failover
|
||||
Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgresq0 in 10 seconds
|
||||
Then I receive a response code 200
|
||||
And postgres0 is a leader after 15 seconds
|
||||
And replication works from postgres0 to postgres1 after 25 seconds
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
import psycopg2 as pg
|
||||
|
||||
from behave import step, then
|
||||
from time import sleep, time
|
||||
|
||||
|
||||
@step('I start {name:w}')
|
||||
def start_patroni(context, name):
|
||||
return context.pctl.start(name)
|
||||
|
||||
|
||||
@step('I shut down {name:w}')
|
||||
def stop_patroni(context, name):
|
||||
return context.pctl.stop(name)
|
||||
|
||||
|
||||
@step('I kill {name:w}')
|
||||
def kill_patroni(context, name):
|
||||
return context.pctl.stop(name, kill=True)
|
||||
|
||||
|
||||
@step('I add the table {table_name:w} to {pg_name:w}')
|
||||
def add_table(context, table_name, pg_name):
|
||||
# parse the configuration file and get the port
|
||||
try:
|
||||
context.pctl.query(pg_name, "CREATE TABLE {0}()".format(table_name))
|
||||
except pg.Error as e:
|
||||
assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e)
|
||||
|
||||
|
||||
@then('Table {table_name:w} is present on {pg_name:w} after {max_replication_delay:d} seconds')
|
||||
def table_is_present_on(context, table_name, pg_name, max_replication_delay):
|
||||
for _ in range(int(max_replication_delay)):
|
||||
if context.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
|
||||
break
|
||||
sleep(1)
|
||||
else:
|
||||
assert False,\
|
||||
"Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay)
|
||||
|
||||
|
||||
@then('{pg_name:w} role is the {pg_role:w} after {max_promotion_timeout:d} seconds')
|
||||
def check_role(context, pg_name, pg_role, max_promotion_timeout):
|
||||
if not context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)):
|
||||
assert False,\
|
||||
"{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
|
||||
|
||||
|
||||
@step('replication works from {master:w} to {replica:w} after {time_limit:d} seconds')
|
||||
@then('replication works from {master:w} to {replica:w} after {time_limit:d} seconds')
|
||||
def replication_works(context, master, replica, time_limit):
|
||||
context.execute_steps(u"""
|
||||
When I add the table test_{0} to {1}
|
||||
Then table test_{0} is present on {2} after {3} seconds
|
||||
""".format(int(time()), master, replica, time_limit))
|
||||
@@ -1,17 +0,0 @@
|
||||
from behave import step, then
|
||||
|
||||
|
||||
@step('I configure and start {name:w} with a tag {tag_name:w} {tag_value:w}')
|
||||
def start_patroni_with_a_name_value_tag(context, name, tag_name, tag_value):
|
||||
return context.pctl.start(name, tags={tag_name: tag_value})
|
||||
|
||||
|
||||
@then('There is a label with "{content:w}" in {name:w} data directory')
|
||||
def check_label(context, content, name):
|
||||
label = context.pctl.read_label(name)
|
||||
assert label == content, "{0} is not equal to {1}".format(label, content)
|
||||
|
||||
|
||||
@step('I create label with "{content:w}" in {name:w} data directory')
|
||||
def write_label(context, content, name):
|
||||
context.pctl.write_label(name, content)
|
||||
@@ -1,101 +0,0 @@
|
||||
import parse
|
||||
import pytz
|
||||
import requests
|
||||
import time
|
||||
|
||||
from behave import register_type, step, then
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
|
||||
@parse.with_pattern(r'https?://(?:\w|\.|:|/)+')
|
||||
def parse_url(text):
|
||||
return text
|
||||
|
||||
|
||||
@parse.with_pattern(r'(?:\w+=(?:\w|\.|:|-|\+|\s)+,?)+')
|
||||
def parse_data(text):
|
||||
return text
|
||||
|
||||
|
||||
register_type(url=parse_url, data=parse_data)
|
||||
|
||||
|
||||
# there is no way we can find out if the node has already
|
||||
# started as a leader without checking the DCS. We cannot
|
||||
# just rely on the database availability, since there is
|
||||
# a short gap between the time PostgreSQL becomes available
|
||||
# and Patroni assuming the leader role.
|
||||
@step('{name:w} is a leader after {time_limit:d} seconds')
|
||||
@then('{name:w} is a leader after {time_limit:d} seconds')
|
||||
def is_a_leader(context, name, time_limit):
|
||||
max_time = time.time() + int(time_limit)
|
||||
while (context.etcd_ctl.query("leader") != name):
|
||||
time.sleep(1)
|
||||
if time.time() > max_time:
|
||||
assert False, "{0} is not a leader in etcd after {1} seconds".format(name, time_limit)
|
||||
|
||||
|
||||
@step('I sleep for {value:d} seconds')
|
||||
def sleep_for_n_seconds(context, value):
|
||||
time.sleep(int(value))
|
||||
|
||||
|
||||
@step('I issue a GET request to {url:url}')
|
||||
def do_get(context, url):
|
||||
try:
|
||||
r = requests.get(url)
|
||||
except requests.exceptions.RequestException:
|
||||
context.status_code = None
|
||||
context.response = None
|
||||
else:
|
||||
context.status_code = r.status_code
|
||||
try:
|
||||
context.response = r.json()
|
||||
except ValueError:
|
||||
context.response = r.content.decode('utf-8')
|
||||
|
||||
|
||||
@step('I issue an empty POST request to {url:url}')
|
||||
def do_post_empty(context, url):
|
||||
do_post(context, url, None)
|
||||
|
||||
|
||||
@step('I issue a POST request to {url:url} with {data:data}')
|
||||
def do_post(context, url, data):
|
||||
post_data = {}
|
||||
if data:
|
||||
post_components = data.split(',')
|
||||
for pc in post_components:
|
||||
if '=' in pc:
|
||||
k, v = pc.split('=', 2)
|
||||
post_data[k.strip()] = v.strip()
|
||||
try:
|
||||
r = requests.post(url, json=post_data)
|
||||
except requests.exceptions.RequestException:
|
||||
context.status_code = None
|
||||
context.response = None
|
||||
else:
|
||||
context.status_code = r.status_code
|
||||
try:
|
||||
context.response = r.json()
|
||||
except ValueError:
|
||||
context.response = r.content.decode('utf-8')
|
||||
|
||||
|
||||
@then('I receive a response {component:w} {data}')
|
||||
def check_response(context, component, data):
|
||||
if component == 'code':
|
||||
assert context.status_code == int(data),\
|
||||
"status code {0} != {1}, response: {2}".format(context.status_code, int(data), context.response)
|
||||
elif component == 'text':
|
||||
assert context.response == data.strip('"'), "response {0} does not contain {1}".format(context.response, data)
|
||||
else:
|
||||
assert component in context.response, "{0} is not part of the response".format(component)
|
||||
assert context.response[component] == data, "{0} does not contain {1}".format(component, data)
|
||||
|
||||
|
||||
@step('I issue a scheduled failover at {at_url:url} from {from_host:w} to {to_host:w} in {in_seconds:d} seconds')
|
||||
def scheduled_failover(context, at_url, from_host, to_host, in_seconds):
|
||||
context.execute_steps(u"""
|
||||
Given I issue a POST request to {0}/failover with leader={1},candidate={2},scheduled_at={3}
|
||||
""".format(at_url, from_host, to_host, datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds))))
|
||||
+4
-13
@@ -10,19 +10,18 @@ from patroni.ha import Ha
|
||||
from patroni.postgresql import Postgresql
|
||||
from patroni.utils import setup_signal_handlers, reap_children
|
||||
from patroni.zookeeper import ZooKeeper
|
||||
from .version import __version__
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Patroni(object):
|
||||
class Patroni:
|
||||
|
||||
def __init__(self, config):
|
||||
self.nap_time = config['loop_wait']
|
||||
self.tags = config.get('tags', dict())
|
||||
self.postgresql = Postgresql(config['postgresql'])
|
||||
self.bdr = config.get('bdr', dict())
|
||||
self.postgresql = Postgresql(config['postgresql'], self.bdr)
|
||||
self.dcs = self.get_dcs(self.postgresql.name, config)
|
||||
self.version = __version__
|
||||
self.api = RestApiServer(self, config['restapi'])
|
||||
self.ha = Ha(self)
|
||||
self.next_run = time.time()
|
||||
@@ -31,14 +30,6 @@ class Patroni(object):
|
||||
def nofailover(self):
|
||||
return self.tags.get('nofailover', False)
|
||||
|
||||
@property
|
||||
def replicatefrom(self):
|
||||
return self.tags.get('replicatefrom')
|
||||
|
||||
@property
|
||||
def clonefrom(self):
|
||||
return self.tags.get('clonefrom')
|
||||
|
||||
@staticmethod
|
||||
def get_dcs(name, config):
|
||||
if 'etcd' in config:
|
||||
@@ -72,7 +63,7 @@ def main():
|
||||
setup_signal_handlers()
|
||||
|
||||
if len(sys.argv) < 2 or not os.path.isfile(sys.argv[1]):
|
||||
print('Usage: {0} config.yml'.format(sys.argv[0]))
|
||||
print('Usage: {} config.yml'.format(sys.argv[0]))
|
||||
return
|
||||
|
||||
with open(sys.argv[1], 'r') as f:
|
||||
|
||||
+19
-56
@@ -5,9 +5,6 @@ import logging
|
||||
import psycopg2
|
||||
import socket
|
||||
import time
|
||||
import dateutil
|
||||
import datetime
|
||||
import pytz
|
||||
|
||||
from patroni.exceptions import PostgresConnectionException
|
||||
from patroni.utils import Retry, RetryFailedError
|
||||
@@ -95,7 +92,6 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
def do_GET_patroni(self):
|
||||
response = self.get_postgresql_status(True)
|
||||
response.update(self.get_tags())
|
||||
response['patroni'] = {'version': self.server.patroni.version, 'scope': self.server.patroni.postgresql.scope}
|
||||
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
@@ -104,7 +100,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
|
||||
@check_auth
|
||||
def do_POST_restart(self):
|
||||
status_code = 500
|
||||
status_code = 503
|
||||
data = b'restart failed'
|
||||
try:
|
||||
status, msg = self.server.patroni.ha.restart()
|
||||
@@ -143,7 +139,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
self.wfile.write(data)
|
||||
|
||||
def poll_failover_result(self, leader, member):
|
||||
for _ in range(0, 15):
|
||||
for a in range(0, 15):
|
||||
time.sleep(1)
|
||||
try:
|
||||
cluster = self.server.patroni.dcs.get_cluster()
|
||||
@@ -166,7 +162,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
members = [m for m in cluster.members if m.name != cluster.leader.name and m.api_url]
|
||||
if not members:
|
||||
return b'failover is not possible: cluster does not have members except leader'
|
||||
for member, reachable, _, xlog_location, tags in self.server.patroni.ha.fetch_nodes_statuses(members):
|
||||
for member, reachable, in_recovery, xlog_location, tags in self.server.patroni.ha.fetch_nodes_statuses(members):
|
||||
if reachable and not tags.get('nofailover', False):
|
||||
return None
|
||||
return b'failover is not possible: no good candidates have been found'
|
||||
@@ -174,48 +170,18 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
@check_auth
|
||||
def do_POST_failover(self):
|
||||
content_length = int(self.headers.get('content-length', 0))
|
||||
try:
|
||||
request = json.loads(self.rfile.read(content_length).decode('utf-8'))
|
||||
except ValueError:
|
||||
request = {}
|
||||
leader = request.get('leader')
|
||||
member = request.get('member')
|
||||
request = json.loads(self.rfile.read(content_length).decode('utf-8'))
|
||||
leader = request.get('leader', None)
|
||||
member = request.get('member', None)
|
||||
cluster = self.server.patroni.ha.dcs.get_cluster()
|
||||
status_code = 500
|
||||
|
||||
logger.info("received failover request with leader {0} member {1} scheduled_at {2}".
|
||||
format(leader, member, request.get("scheduled_at")))
|
||||
|
||||
data = b''
|
||||
if leader or member:
|
||||
if request.get('scheduled_at'):
|
||||
try:
|
||||
scheduled_at = dateutil.parser.parse(request['scheduled_at'])
|
||||
if scheduled_at.tzinfo is None:
|
||||
data = b'Timezone information is mandatory for scheduled_at'
|
||||
status_code = 400
|
||||
elif scheduled_at < datetime.datetime.now(pytz.utc):
|
||||
data = b'Cannot schedule failover in the past'
|
||||
status_code = 422
|
||||
elif self.server.patroni.dcs.manual_failover(leader, member, scheduled_at):
|
||||
data = b'Failover scheduled'
|
||||
status_code = 200
|
||||
except (ValueError, TypeError):
|
||||
logger.exception('Invalid scheduled failover time: {}'.format(request['scheduled_at']))
|
||||
data = b'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601'
|
||||
status_code = 422
|
||||
status_code = 503
|
||||
data = self.is_failover_possible(cluster, leader, member)
|
||||
if not data:
|
||||
if not self.server.patroni.dcs.manual_failover(leader, member):
|
||||
data = b'failed to write failover key into DCS'
|
||||
else:
|
||||
data = self.is_failover_possible(cluster, leader, member)
|
||||
if not data:
|
||||
if not self.server.patroni.dcs.manual_failover(leader, member):
|
||||
data = b'failed to write failover key into DCS'
|
||||
status_code = 503
|
||||
else:
|
||||
self.server.patroni.dcs.event.set()
|
||||
status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, member)
|
||||
else:
|
||||
status_code = 400
|
||||
data = b'No values given for required parameters leader and member'
|
||||
self.server.patroni.dcs.event.set()
|
||||
status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, member)
|
||||
|
||||
self.send_response(status_code)
|
||||
self.send_header('Content-Type', 'text/html')
|
||||
@@ -262,18 +228,15 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
END,
|
||||
pg_xlog_location_diff(pg_last_xlog_receive_location(), '0/0')::bigint,
|
||||
pg_xlog_location_diff(pg_last_xlog_replay_location(), '0/0')::bigint,
|
||||
to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
|
||||
pg_is_in_recovery() AND pg_is_xlog_replay_paused()""", retry=retry)[0]
|
||||
return {
|
||||
'state': self.server.patroni.postgresql.state,
|
||||
'postmaster_start_time': row[0],
|
||||
'role': 'replica' if row[1] else 'master',
|
||||
'server_version': self.server.patroni.postgresql.server_version,
|
||||
'xlog': ({
|
||||
'received_location': row[3],
|
||||
'replayed_location': row[4],
|
||||
'replayed_timestamp': row[5],
|
||||
'paused': row[6]} if row[1] else {
|
||||
'paused': row[5]} if row[1] else {
|
||||
'location': row[2]
|
||||
})
|
||||
}
|
||||
@@ -287,8 +250,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
def get_tags(self):
|
||||
return {'tags': self.server.patroni.tags}
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
logger.debug("API thread: %s - - [%s] %s", self.client_address[0], self.log_date_time_string(), fmt % args)
|
||||
def log_message(self, format, *args):
|
||||
logger.debug("API thread: " + format % args)
|
||||
|
||||
|
||||
class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
|
||||
@@ -305,12 +268,12 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
|
||||
# wrap socket with ssl if 'certfile' is defined in a config.yaml
|
||||
# Sometime it's also needed to pass reference to a 'keyfile'.
|
||||
options = {option: config[option] for option in ['certfile', 'keyfile'] if option in config}
|
||||
if options.get('certfile'):
|
||||
if options.get('certfile', None):
|
||||
import ssl
|
||||
self.socket = ssl.wrap_socket(self.socket, server_side=True, **options)
|
||||
protocol = 'https'
|
||||
|
||||
self.connection_string = '{0}://{1}/patroni'.format(protocol, config.get('connect_address', config['listen']))
|
||||
self.connection_string = '{}://{}/patroni'.format(protocol, config.get('connect_address', config['listen']))
|
||||
|
||||
self.patroni = patroni
|
||||
self.daemon = True
|
||||
@@ -318,7 +281,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
|
||||
def query(self, sql, *params):
|
||||
cursor = None
|
||||
try:
|
||||
with self.patroni.postgresql.connection().cursor() as cursor:
|
||||
with self.patroni.postgresql.connection('postgres').cursor() as cursor:
|
||||
cursor.execute(sql, params)
|
||||
return [r for r in cursor]
|
||||
except psycopg2.Error as e:
|
||||
|
||||
@@ -4,9 +4,10 @@ from threading import Lock, Thread
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AsyncExecutor(object):
|
||||
class AsyncExecutor:
|
||||
|
||||
def __init__(self):
|
||||
Lock.__init__(self)
|
||||
self._busy = False
|
||||
self._thread_lock = Lock()
|
||||
self._scheduled_action = None
|
||||
@@ -50,5 +51,5 @@ class AsyncExecutor(object):
|
||||
def __enter__(self):
|
||||
self._thread_lock.acquire()
|
||||
|
||||
def __exit__(self, *args):
|
||||
def __exit__(self, type, value, traceback):
|
||||
self._thread_lock.release()
|
||||
|
||||
+84
-121
@@ -14,11 +14,8 @@ import datetime
|
||||
from prettytable import PrettyTable
|
||||
from six.moves.urllib_parse import urlparse
|
||||
import logging
|
||||
import dateutil
|
||||
import tzlocal
|
||||
|
||||
from .etcd import Etcd
|
||||
from .zookeeper import ZooKeeper
|
||||
from .exceptions import PatroniCtlException
|
||||
from .postgresql import parseurl
|
||||
|
||||
@@ -47,24 +44,24 @@ def parse_dcs(dcs):
|
||||
parsed = urlparse('//' + dcs)
|
||||
|
||||
if scheme == '':
|
||||
default_schemes = {'2181': 'zookeeper', '8181': 'exhibitor', '8500': 'consul'}
|
||||
default_schemes = {'2181': 'zookeeper', '8500': 'consul'}
|
||||
scheme = default_schemes.get(str(parsed.port), 'etcd')
|
||||
|
||||
port = parsed.port
|
||||
if port is None:
|
||||
default_ports = {'consul': 8500, 'zookeeper': 2181, 'exhibitor': 8181}
|
||||
default_ports = {'consul': 8500, 'zookeeper': 2181}
|
||||
port = default_ports.get(str(scheme), 4001)
|
||||
|
||||
return {'scheme': str(scheme), 'hostname': str(parsed.hostname), 'port': int(port)}
|
||||
|
||||
|
||||
def load_config(path, dcs):
|
||||
logging.debug('Loading configuration from file %s', path)
|
||||
logging.debug('Loading configuration from file {}'.format(path))
|
||||
config = dict()
|
||||
try:
|
||||
with open(path, 'rb') as fd:
|
||||
config = yaml.safe_load(fd)
|
||||
except (IOError, yaml.YAMLError):
|
||||
except:
|
||||
logging.exception('Could not load configuration file')
|
||||
|
||||
if dcs:
|
||||
@@ -77,14 +74,15 @@ def load_config(path, dcs):
|
||||
|
||||
def store_config(config, path):
|
||||
dir_path = os.path.dirname(path)
|
||||
if dir_path and not os.path.isdir(dir_path):
|
||||
os.makedirs(dir_path)
|
||||
if dir_path:
|
||||
if not os.path.isdir(dir_path):
|
||||
os.makedirs(dir_path)
|
||||
with open(path, 'w') as fd:
|
||||
yaml.dump(config, fd)
|
||||
|
||||
|
||||
option_config_file = click.option('--config-file', '-c', help='Configuration file', default=CONFIG_FILE_PATH)
|
||||
option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, json)', default='pretty')
|
||||
option_format = click.option('--format', '-f', help='Output format (pretty, json)', default='pretty')
|
||||
option_dcs = click.option('--dcs', '-d', help='Use this DCS', envvar='DCS')
|
||||
option_watchrefresh = click.option('-w', '--watch', type=float, help='Auto update the screen every X seconds')
|
||||
option_watch = click.option('-W', is_flag=True, help='Auto update the screen every 2 seconds')
|
||||
@@ -104,28 +102,20 @@ def get_dcs(config, scope):
|
||||
scheme, hostname, port = map(config.get('dcs', {}).get, ('scheme', 'hostname', 'port'))
|
||||
|
||||
if scheme == 'etcd':
|
||||
return Etcd(name=scope, config={'scope': scope, 'host': '{0}:{1}'.format(hostname, port)})
|
||||
|
||||
if scheme == 'zookeeper':
|
||||
return ZooKeeper(name=scope, config={'scope': scope, 'hosts': [hostname], 'port': port})
|
||||
|
||||
if scheme == 'exhibitor':
|
||||
return ZooKeeper(name=scope, config={'scope': scope, 'exhibitor': {'hosts': [hostname], 'port': port}})
|
||||
return Etcd(name=scope, config={'scope': scope, 'host': '{}:{}'.format(hostname, port)})
|
||||
|
||||
raise PatroniCtlException('Can not find suitable configuration of distributed configuration store')
|
||||
|
||||
|
||||
def post_patroni(member, endpoint, content, headers=None):
|
||||
def post_patroni(member, endpoint, content, headers={'Content-Type': 'application/json'}):
|
||||
url = urlparse(member.api_url)
|
||||
logging.debug(url)
|
||||
return requests.post('{0}://{1}/{2}'.format(url.scheme, url.netloc, endpoint),
|
||||
headers=headers or {'Content-Type': 'application/json'},
|
||||
return requests.post('{}://{}/{}'.format(url.scheme, url.netloc, endpoint), headers=headers,
|
||||
data=json.dumps(content), timeout=60)
|
||||
|
||||
|
||||
def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True, delimiter='\t'):
|
||||
rows = rows or []
|
||||
if fmt == 'pretty':
|
||||
def print_output(columns, rows=[], alignment=None, format='pretty', header=True, delimiter='\t'):
|
||||
if format == 'pretty':
|
||||
t = PrettyTable(columns)
|
||||
for k, v in (alignment or {}).items():
|
||||
t.align[k] = v
|
||||
@@ -134,18 +124,18 @@ def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True,
|
||||
click.echo(t)
|
||||
return
|
||||
|
||||
if fmt == 'json':
|
||||
if format == 'json':
|
||||
elements = list()
|
||||
for r in rows:
|
||||
elements.append(dict(zip(columns, r)))
|
||||
|
||||
click.echo(json.dumps(elements))
|
||||
|
||||
if fmt == 'tsv':
|
||||
if format == 'tsv':
|
||||
if columns is not None and header:
|
||||
click.echo(delimiter.join(columns) + '\n')
|
||||
|
||||
for r in rows:
|
||||
for r in rows or []:
|
||||
c = [str(c) for c in r]
|
||||
click.echo(delimiter.join(c))
|
||||
|
||||
@@ -178,8 +168,8 @@ def watching(w, watch, max_count=None, clear=True):
|
||||
yield 0
|
||||
|
||||
|
||||
def build_connect_parameters(conn_url, connect_parameters=None):
|
||||
params = (connect_parameters or {}).copy()
|
||||
def build_connect_parameters(conn_url, connect_parameters={}):
|
||||
params = connect_parameters.copy()
|
||||
parsed = parseurl(conn_url)
|
||||
params['host'] = parsed['host']
|
||||
params['port'] = parsed['port']
|
||||
@@ -210,12 +200,12 @@ def get_any_member(cluster, role='master', member=None):
|
||||
return None
|
||||
|
||||
|
||||
def get_cursor(cluster, role='master', member=None, connect_parameters=None):
|
||||
def get_cursor(cluster, role='master', member=None, connect_parameters={}):
|
||||
member = get_any_member(cluster=cluster, role=role, member=member)
|
||||
if member is None:
|
||||
return None
|
||||
|
||||
params = build_connect_parameters(member.conn_url, connect_parameters)
|
||||
params = build_connect_parameters(member.conn_url, connect_parameters=connect_parameters)
|
||||
|
||||
conn = psycopg2.connect(**params)
|
||||
conn.autocommit = True
|
||||
@@ -247,23 +237,21 @@ def dsn(cluster_name, config_file, dcs, role, member):
|
||||
if member is None and role is None:
|
||||
role = 'master'
|
||||
|
||||
_, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
|
||||
config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
|
||||
m = get_any_member(cluster=cluster, role=role, member=member)
|
||||
if m is None:
|
||||
raise PatroniCtlException('Can not find a suitable member')
|
||||
|
||||
params = build_connect_parameters(m.conn_url)
|
||||
click.echo('host={host} port={port}'.format(**params))
|
||||
click.echo('host={} port={}'.format(params['host'], params['port']))
|
||||
|
||||
|
||||
@ctl.command('query', help='Query a Patroni PostgreSQL member')
|
||||
@click.argument('cluster_name')
|
||||
@option_config_file
|
||||
@option_format
|
||||
@click.option('--format', 'fmt', help='Output format (pretty, json)', default='tsv')
|
||||
@click.option('--file', '-f', 'p_file', help='Execute the SQL commands from this file', type=click.File('rb'))
|
||||
@click.option('--password', help='force password prompt', is_flag=True)
|
||||
@click.option('-U', '--username', help='database user name', type=str)
|
||||
@click.option('--format', help='Output format (pretty, json)', default='tsv')
|
||||
@click.option('--file', '-f', help='Execute the SQL commands from this file', type=click.File('rb'))
|
||||
@option_dcs
|
||||
@option_watch
|
||||
@option_watchrefresh
|
||||
@@ -272,7 +260,6 @@ def dsn(cluster_name, config_file, dcs, role, member):
|
||||
@click.option('--member', '-m', help='Query a specific member', type=str)
|
||||
@click.option('--delimiter', help='The column delimiter', default='\t')
|
||||
@click.option('--command', '-c', help='The SQL commands to execute')
|
||||
@click.option('-d', '--dbname', help='database name to connect to', type=str)
|
||||
def query(
|
||||
cluster_name,
|
||||
config_file,
|
||||
@@ -283,57 +270,42 @@ def query(
|
||||
watch,
|
||||
delimiter,
|
||||
command,
|
||||
p_file,
|
||||
password,
|
||||
username,
|
||||
dbname,
|
||||
fmt='tsv',
|
||||
file,
|
||||
format='tsv',
|
||||
):
|
||||
if role is not None and member is not None:
|
||||
raise PatroniCtlException('--role and --member are mutually exclusive options')
|
||||
if member is None and role is None:
|
||||
role = 'master'
|
||||
|
||||
if p_file is not None and command is not None:
|
||||
if file is not None and command is not None:
|
||||
raise PatroniCtlException('--file and --command are mutually exclusive options')
|
||||
|
||||
if p_file is None and command is None:
|
||||
raise PatroniCtlException('You need to specify either --command or --file')
|
||||
|
||||
connect_parameters = dict()
|
||||
if username:
|
||||
connect_parameters['user'] = username
|
||||
if password:
|
||||
connect_parameters['password'] = click.prompt('Password', hide_input=True, type=str)
|
||||
if dbname:
|
||||
connect_parameters['database'] = dbname
|
||||
|
||||
if p_file is not None:
|
||||
command = p_file.read()
|
||||
if file is not None:
|
||||
command = file.read()
|
||||
|
||||
config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
|
||||
|
||||
cursor = None
|
||||
for _ in watching(w, watch, clear=False):
|
||||
|
||||
output, cursor = query_member(cluster=cluster, cursor=cursor, member=member, role=role, command=command,
|
||||
connect_parameters=connect_parameters)
|
||||
print_output(None, output, fmt=fmt, delimiter=delimiter)
|
||||
output, cursor = query_member(cluster=cluster, cursor=cursor, member=member, role=role, command=command)
|
||||
print_output(None, output, format=format, delimiter=delimiter)
|
||||
|
||||
if cursor is None:
|
||||
cluster = dcs.get_cluster()
|
||||
|
||||
|
||||
def query_member(cluster, cursor, member, role, command, connect_parameters=None):
|
||||
def query_member(cluster, cursor, member, role, command):
|
||||
try:
|
||||
if cursor is None:
|
||||
cursor = get_cursor(cluster, role=role, member=member, connect_parameters=connect_parameters)
|
||||
cursor = get_cursor(cluster, role=role, member=member)
|
||||
|
||||
if cursor is None:
|
||||
if role is None:
|
||||
message = 'No connection to member {0} is available'.format(member)
|
||||
message = 'No connection to member {} is available'.format(member)
|
||||
else:
|
||||
message = 'No connection to role={0} is available'.format(role)
|
||||
message = 'No connection to role={} is available'.format(role)
|
||||
logging.debug(message)
|
||||
return [[timestamp(0), message]], None
|
||||
|
||||
@@ -352,7 +324,7 @@ def query_member(cluster, cursor, member, role, command, connect_parameters=None
|
||||
cursor.connection.close()
|
||||
message = oe.pgcode or oe.pgerror or str(oe)
|
||||
message = message.replace('\n', ' ')
|
||||
return [[timestamp(0), 'ERROR, SQLSTATE: {0}'.format(message)]], None
|
||||
return [[timestamp(0), 'ERROR, SQLSTATE: {}'.format(message)]], None
|
||||
|
||||
|
||||
@ctl.command('remove', help='Remove cluster from DCS')
|
||||
@@ -360,13 +332,13 @@ def query_member(cluster, cursor, member, role, command, connect_parameters=None
|
||||
@option_config_file
|
||||
@option_format
|
||||
@option_dcs
|
||||
def remove(config_file, cluster_name, fmt, dcs):
|
||||
def remove(config_file, cluster_name, format, dcs):
|
||||
config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
|
||||
|
||||
if not isinstance(dcs, Etcd):
|
||||
raise PatroniCtlException('We have not implemented this for DCS of type {0}'.format(type(dcs)))
|
||||
raise PatroniCtlException('We have not implemented this for DCS of type {}'.format(type(dcs)))
|
||||
|
||||
output_members(cluster, fmt=fmt)
|
||||
output_members(cluster, format=format)
|
||||
|
||||
confirm = click.prompt('Please confirm the cluster name to remove', type=str)
|
||||
if confirm != cluster_name:
|
||||
@@ -374,17 +346,17 @@ def remove(config_file, cluster_name, fmt, dcs):
|
||||
|
||||
message = 'Yes I am aware'
|
||||
confirm = \
|
||||
click.prompt('You are about to remove all information in DCS for {0}, please type: "{1}"'.format(cluster_name,
|
||||
click.prompt('You are about to remove all information in DCS for {}, please type: "{}"'.format(cluster_name,
|
||||
message), type=str)
|
||||
if message != confirm:
|
||||
raise PatroniCtlException('You did not exactly type "{0}"'.format(message))
|
||||
raise PatroniCtlException('You did not exactly type "{}"'.format(message))
|
||||
|
||||
if cluster.leader:
|
||||
confirm = click.prompt('This cluster currently is healthy. Please specify the master name to continue')
|
||||
if confirm != cluster.leader.name:
|
||||
raise PatroniCtlException('You did not specify the current master of the cluster')
|
||||
|
||||
dcs.client.delete(dcs.client_path(''), recursive=True)
|
||||
dcs.client.delete(dcs._base_path, recursive=True)
|
||||
|
||||
|
||||
def wait_for_leader(dcs, timeout=30):
|
||||
@@ -406,25 +378,25 @@ def empty_post_to_members(cluster, member_names, force, endpoint):
|
||||
for m in cluster.members:
|
||||
candidates[m.name] = m
|
||||
|
||||
if not member_names:
|
||||
member_names = [click.prompt('Which member do you want to {0} [{1}]?'.format(endpoint,
|
||||
if len(member_names) == 0:
|
||||
member_names = [click.prompt('Which member do you want to {} [{}]?'.format(endpoint,
|
||||
', '.join(candidates.keys())), type=str, default='')]
|
||||
|
||||
for mn in member_names:
|
||||
if mn not in candidates.keys():
|
||||
raise PatroniCtlException('{0} is not a member of cluster'.format(mn))
|
||||
raise PatroniCtlException('{} is not a member of cluster'.format(mn))
|
||||
|
||||
if not force:
|
||||
confirm = click.confirm('Are you sure you want to {0} members {1}?'.format(endpoint, ', '.join(member_names)))
|
||||
confirm = click.confirm('Are you sure you want to {} members {}?'.format(endpoint, ', '.join(member_names)))
|
||||
if not confirm:
|
||||
raise PatroniCtlException('Aborted {0}'.format(endpoint))
|
||||
raise PatroniCtlException('Aborted {}'.format(endpoint))
|
||||
|
||||
for mn in member_names:
|
||||
r = post_patroni(candidates[mn], endpoint, '')
|
||||
if r.status_code != 200:
|
||||
click.echo('{0} failed for member {1}, status code={2}, ({3})'.format(endpoint, mn, r.status_code, r.text))
|
||||
click.echo('{} failed for member {}, status code={}, ({})'.format(endpoint, mn, r.status_code, r.text))
|
||||
else:
|
||||
click.echo('Succesful {0} on member {1}'.format(endpoint, mn))
|
||||
click.echo('Succesful {} on member {}'.format(endpoint, mn))
|
||||
|
||||
|
||||
def ctl_load_config(cluster_name, config_file, dcs):
|
||||
@@ -440,21 +412,21 @@ def ctl_load_config(cluster_name, config_file, dcs):
|
||||
@click.argument('member_names', nargs=-1)
|
||||
@click.option('--role', '-r', help='Restart only members with this role', default='any',
|
||||
type=click.Choice(['master', 'replica', 'any']))
|
||||
@click.option('--any', 'p_any', help='Restart a single member only', is_flag=True)
|
||||
@click.option('--any', help='Restart a single member only', is_flag=True)
|
||||
@option_config_file
|
||||
@option_force
|
||||
@option_dcs
|
||||
def restart(cluster_name, member_names, config_file, dcs, force, role, p_any):
|
||||
def restart(cluster_name, member_names, config_file, dcs, force, role, any):
|
||||
config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
|
||||
|
||||
role_names = [m.name for m in get_all_members(cluster=cluster, role=role)]
|
||||
|
||||
if member_names:
|
||||
if len(member_names) > 0:
|
||||
member_names = list(set(member_names) & set(role_names))
|
||||
else:
|
||||
member_names = role_names
|
||||
|
||||
if p_any:
|
||||
if any:
|
||||
random.shuffle(member_names)
|
||||
member_names = member_names[:1]
|
||||
|
||||
@@ -477,12 +449,10 @@ def reinit(cluster_name, member_names, config_file, dcs, force):
|
||||
@click.argument('cluster_name')
|
||||
@click.option('--master', help='The name of the current master', default=None)
|
||||
@click.option('--candidate', help='The name of the candidate', default=None)
|
||||
@click.option('--scheduled', help='Timestamp of a scheduled failover in unambiguous format (e.g. ISO 8601)',
|
||||
default=None)
|
||||
@click.option('--force', is_flag=True)
|
||||
@option_config_file
|
||||
@option_dcs
|
||||
def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled):
|
||||
def failover(config_file, cluster_name, master, candidate, force, dcs):
|
||||
"""
|
||||
We want to trigger a failover for the specified cluster name.
|
||||
|
||||
@@ -502,13 +472,13 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
|
||||
master = click.prompt('Master', type=str, default=cluster.leader.member.name)
|
||||
|
||||
if cluster.leader.member.name != master:
|
||||
raise PatroniCtlException('Member {0} is not the leader of cluster {1}'.format(master, cluster_name))
|
||||
raise PatroniCtlException('Member {} is not the leader of cluster {}'.format(master, cluster_name))
|
||||
|
||||
candidate_names = [str(m.name) for m in cluster.members if m.name != master]
|
||||
# We sort the names for consistent output to the client
|
||||
candidate_names.sort()
|
||||
|
||||
if not candidate_names:
|
||||
if len(candidate_names) == 0:
|
||||
raise PatroniCtlException('No candidates found to failover to')
|
||||
|
||||
if candidate is None and not force:
|
||||
@@ -518,26 +488,7 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
|
||||
raise PatroniCtlException('Failover target and source are the same.')
|
||||
|
||||
if candidate and candidate not in candidate_names:
|
||||
raise PatroniCtlException('Member {0} does not exist in cluster {1}'.format(candidate, cluster_name))
|
||||
|
||||
if scheduled is None and not force:
|
||||
scheduled = click.prompt('When should the failover take place (e.g. 2015-10-01T14:30) ', type=str,
|
||||
default='now')
|
||||
|
||||
if (scheduled or 'now') == 'now':
|
||||
scheduled_at = None
|
||||
else:
|
||||
try:
|
||||
scheduled_at = dateutil.parser.parse(scheduled)
|
||||
if scheduled_at.tzinfo is None:
|
||||
scheduled_at = tzlocal.get_localzone().localize(scheduled_at)
|
||||
except (ValueError, TypeError):
|
||||
message = 'Unable to parse scheduled timestamp ({}). It should be in an unambiguous format (e.g. ISO 8601)'
|
||||
raise PatroniCtlException(message.format(scheduled))
|
||||
scheduled_at = scheduled_at.isoformat()
|
||||
|
||||
failover_value = {'leader': master, 'member': candidate, 'scheduled_at': scheduled_at}
|
||||
logging.debug(failover_value)
|
||||
raise PatroniCtlException('Member {} does not exist in cluster {}'.format(candidate, cluster_name))
|
||||
|
||||
# By now we have established that the leader exists and the candidate exists
|
||||
click.echo('Current cluster topology')
|
||||
@@ -545,33 +496,44 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
|
||||
|
||||
if not force:
|
||||
a = \
|
||||
click.confirm('Are you sure you want to failover cluster {0}, demoting current master {1}?'.format(
|
||||
click.confirm('Are you sure you want to failover cluster {}, demoting current master {}?'.format(
|
||||
cluster_name, master))
|
||||
if not a:
|
||||
raise PatroniCtlException('Aborting failover')
|
||||
|
||||
failover_value = '{}:{}'.format(master, candidate or '')
|
||||
|
||||
t_started = time.time()
|
||||
r = None
|
||||
try:
|
||||
r = post_patroni(cluster.leader.member, 'failover', failover_value)
|
||||
r = post_patroni(cluster.leader.member, 'failover', {'leader': master, 'member': candidate or ''})
|
||||
if r.status_code == 200:
|
||||
logging.debug(r)
|
||||
logging.debug(r.text)
|
||||
cluster = dcs.get_cluster()
|
||||
logging.debug(cluster)
|
||||
click.echo('{0} {1}'.format(timestamp(), r.text))
|
||||
click.echo(timestamp() + ' Failing over to new leader: {}'.format(cluster.leader.member.name))
|
||||
else:
|
||||
click.echo('Failover failed, details: {0}, {1}'.format(r.status_code, r.text))
|
||||
click.echo('Failover failed, details: {}, {}'.format(r.status_code, r.text))
|
||||
return
|
||||
except:
|
||||
logging.exception(r)
|
||||
logging.warning('Failing over to DCS')
|
||||
click.echo(timestamp() + ' Could not failover using Patroni api, falling back to DCS')
|
||||
click.echo(timestamp() + ' Initializing failover from master {0}'.format(master))
|
||||
dcs.manual_failover(leader=master, member=candidate, scheduled_at=failover_value)
|
||||
dcs.set_failover_value(failover_value)
|
||||
click.echo(timestamp() + ' Initialized failover from master {}'.format(master))
|
||||
# The failover process should within a minute update the failover key, we will keep watching it until it changes
|
||||
# or we timeout
|
||||
cluster = wait_for_leader(dcs, timeout=60)
|
||||
if cluster.leader.member.name == master:
|
||||
click.echo('Failover failed, master did not change after {:0.1f} seconds'.format(time.time() - t_started))
|
||||
return
|
||||
|
||||
click.echo(timestamp() + ' Failover completed in {:0.1f} seconds, new leader is {}'.format(time.time() - t_started,
|
||||
str(cluster.leader.member.name)))
|
||||
output_members(cluster, name=cluster_name)
|
||||
|
||||
|
||||
def output_members(cluster, name=None, fmt='pretty'):
|
||||
def output_members(cluster, name=None, format='pretty'):
|
||||
rows = []
|
||||
logging.debug(cluster)
|
||||
leader_name = None
|
||||
@@ -591,9 +553,10 @@ def output_members(cluster, name=None, fmt='pretty'):
|
||||
|
||||
host = build_connect_parameters(m.conn_url)['host']
|
||||
|
||||
xlog_location = m.data.get('xlog_location') or 0
|
||||
lag = ''
|
||||
if (xlog_location_cluster >= xlog_location):
|
||||
xlog_location = m.data.get('xlog_location')
|
||||
if xlog_location is None or (xlog_location_cluster < xlog_location):
|
||||
lag = ''
|
||||
else:
|
||||
lag = round((xlog_location_cluster - xlog_location)/1024/1024)
|
||||
|
||||
rows.append([
|
||||
@@ -615,7 +578,7 @@ def output_members(cluster, name=None, fmt='pretty'):
|
||||
]
|
||||
alignment = {'Cluster': 'l', 'Member': 'l', 'Host': 'l', 'Lag in MB': 'r'}
|
||||
|
||||
print_output(columns, rows, alignment, fmt)
|
||||
print_output(columns, rows, alignment, format)
|
||||
|
||||
|
||||
@ctl.command('list', help='List the Patroni members for a given Patroni')
|
||||
@@ -625,8 +588,8 @@ def output_members(cluster, name=None, fmt='pretty'):
|
||||
@option_watch
|
||||
@option_watchrefresh
|
||||
@option_dcs
|
||||
def members(config_file, cluster_names, fmt, watch, w, dcs):
|
||||
if not cluster_names:
|
||||
def members(config_file, cluster_names, format, watch, w, dcs):
|
||||
if len(cluster_names) == 0:
|
||||
logging.warning('Listing members: No cluster names were provided')
|
||||
return
|
||||
|
||||
@@ -635,7 +598,7 @@ def members(config_file, cluster_names, fmt, watch, w, dcs):
|
||||
dcs = get_dcs(config, cn)
|
||||
|
||||
for _ in watching(w, watch):
|
||||
output_members(dcs.get_cluster(), name=cn, fmt=fmt)
|
||||
output_members(dcs.get_cluster(), name=cn, format=format)
|
||||
|
||||
|
||||
def timestamp(precision=6):
|
||||
|
||||
+17
-62
@@ -1,8 +1,8 @@
|
||||
import abc
|
||||
import json
|
||||
import dateutil
|
||||
|
||||
from collections import namedtuple
|
||||
from patroni.exceptions import DCSError
|
||||
from six.moves.urllib_parse import urlparse, urlunparse, parse_qsl
|
||||
from threading import Event, Lock
|
||||
|
||||
@@ -51,26 +51,22 @@ class Member(namedtuple('Member', 'index,name,session,data')):
|
||||
else:
|
||||
try:
|
||||
data = json.loads(data)
|
||||
except (TypeError, ValueError):
|
||||
except:
|
||||
data = {}
|
||||
return Member(index, name, session, data)
|
||||
|
||||
@property
|
||||
def conn_url(self):
|
||||
return self.data.get('conn_url')
|
||||
return self.data.get('conn_url', None)
|
||||
|
||||
@property
|
||||
def api_url(self):
|
||||
return self.data.get('api_url')
|
||||
return self.data.get('api_url', None)
|
||||
|
||||
@property
|
||||
def nofailover(self):
|
||||
return self.data.get('tags', {}).get('nofailover', False)
|
||||
|
||||
@property
|
||||
def replicatefrom(self):
|
||||
return self.data.get('tags', {}).get('replicatefrom')
|
||||
|
||||
|
||||
class Leader(namedtuple('Leader', 'index,session,member')):
|
||||
|
||||
@@ -89,44 +85,12 @@ class Leader(namedtuple('Leader', 'index,session,member')):
|
||||
return self.member.conn_url
|
||||
|
||||
|
||||
class Failover(namedtuple('Failover', 'index,leader,member,scheduled_at')):
|
||||
class Failover(namedtuple('Failover', 'index,leader,member')):
|
||||
|
||||
"""
|
||||
>>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader"}'))
|
||||
True
|
||||
>>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader", "member": "cluster:member"}'))
|
||||
True
|
||||
>>> Failover.from_node(1, 'null') is None
|
||||
True
|
||||
>>> n = '{"leader": "cluster_leader", "member": "cluster:member", "scheduled_at": "2016-01-14T10:09:57.1394Z"}'
|
||||
>>> 'tzinfo=' in str(Failover.from_node(1, n))
|
||||
True
|
||||
>>> Failover.from_node(1, None) is None
|
||||
True
|
||||
>>> Failover.from_node(1, '{}') is None
|
||||
True
|
||||
>>> 'abc' in Failover.from_node(1, 'abc:def')
|
||||
True
|
||||
"""
|
||||
@staticmethod
|
||||
def from_node(index, value):
|
||||
if not value:
|
||||
return None
|
||||
|
||||
try:
|
||||
data = json.loads(value)
|
||||
if not data:
|
||||
return None
|
||||
except ValueError:
|
||||
t = [a.strip() for a in value.split(':')]
|
||||
leader = t[0]
|
||||
candidate = t[1] if len(t) > 1 else None
|
||||
return Failover(index, leader, candidate, None) if leader or candidate else None
|
||||
|
||||
if data.get('scheduled_at'):
|
||||
data['scheduled_at'] = dateutil.parser.parse(data['scheduled_at'])
|
||||
|
||||
return Failover(index, data.get('leader'), data.get('member'), data.get('scheduled_at'))
|
||||
t = [a.strip() for a in value.split(':')] + ['']
|
||||
return Failover(index, t[0], t[1]) if t[0] or t[1] else None
|
||||
|
||||
|
||||
class Cluster(namedtuple('Cluster', 'initialize,leader,last_leader_operation,members,failover')):
|
||||
@@ -143,14 +107,8 @@ class Cluster(namedtuple('Cluster', 'initialize,leader,last_leader_operation,mem
|
||||
def is_unlocked(self):
|
||||
return not (self.leader and self.leader.name)
|
||||
|
||||
def has_member(self, member_name):
|
||||
return any(m for m in self.members if m.name == member_name)
|
||||
|
||||
def get_member(self, member_name):
|
||||
return ([m for m in self.members if m.name == member_name] or [None])[0]
|
||||
|
||||
|
||||
class AbstractDCS(object):
|
||||
class AbstractDCS:
|
||||
|
||||
__metaclass__ = abc.ABCMeta
|
||||
|
||||
@@ -168,7 +126,7 @@ class AbstractDCS(object):
|
||||
i.e.: `zookeeper` for zookeeper, `etcd` for etcd, etc...
|
||||
"""
|
||||
self._name = name
|
||||
self._namespace = '/{0}'.format(config.get('namespace', '/service/').strip('/'))
|
||||
self._namespace = '/{}'.format(config.get('namespace', '/service/').strip('/'))
|
||||
self._base_path = '/'.join([self._namespace, config['scope']])
|
||||
|
||||
self._cluster = None
|
||||
@@ -258,18 +216,15 @@ class AbstractDCS(object):
|
||||
def set_failover_value(self, value, index=None):
|
||||
"""Create or update `/failover` key"""
|
||||
|
||||
def manual_failover(self, leader, member, scheduled_at=None, index=None):
|
||||
failover_value = dict()
|
||||
if leader:
|
||||
failover_value['leader'] = leader
|
||||
def manual_failover(self, leader, member, index=None):
|
||||
return self.set_failover_value(leader + (':' + member if member else ''), index)
|
||||
|
||||
if member:
|
||||
failover_value['member'] = member
|
||||
|
||||
if scheduled_at:
|
||||
failover_value['scheduled_at'] = scheduled_at.isoformat()
|
||||
|
||||
return self.set_failover_value(json.dumps(failover_value), index)
|
||||
def current_leader(self):
|
||||
try:
|
||||
cluster = self.get_cluster()
|
||||
return None if cluster.is_unlocked() else cluster.leader
|
||||
except DCSError:
|
||||
return None
|
||||
|
||||
@abc.abstractmethod
|
||||
def touch_member(self, connection_string, ttl=None):
|
||||
|
||||
+24
-64
@@ -14,7 +14,6 @@ from patroni.dcs import AbstractDCS, Cluster, Failover, Leader, Member
|
||||
from patroni.exceptions import DCSError
|
||||
from patroni.utils import Retry, RetryFailedError, sleep
|
||||
from requests.exceptions import RequestException
|
||||
from six.moves.http_client import HTTPException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -50,57 +49,11 @@ class Client(etcd.Client):
|
||||
self._update_machines_cache = True
|
||||
return [self._base_uri]
|
||||
|
||||
def _do_http_request(self, request_executor, method, url, fields=None, **kwargs):
|
||||
try:
|
||||
response = request_executor(method, url, fields=fields, **kwargs)
|
||||
response.data.decode('utf-8')
|
||||
self._check_cluster_id(response)
|
||||
except (urllib3.exceptions.HTTPError, HTTPException, socket.error) as e:
|
||||
if (isinstance(fields, dict) and fields.get("wait") == "true" and
|
||||
isinstance(e, urllib3.exceptions.ReadTimeoutError)):
|
||||
logger.debug("Watch timed out.")
|
||||
raise etcd.EtcdWatchTimedOut("Watch timed out: {0}".format(e), cause=e)
|
||||
logger.error("Request to server %s failed: %r", self._base_uri, e)
|
||||
logger.info("Reconnection allowed, looking for another server.")
|
||||
self._base_uri = self._next_server(cause=e)
|
||||
response = False
|
||||
return response
|
||||
|
||||
def api_execute(self, path, method, params=None, timeout=None):
|
||||
if not path.startswith('/'):
|
||||
raise ValueError('Path does not start with /')
|
||||
|
||||
if timeout is None:
|
||||
timeout = self.read_timeout
|
||||
|
||||
if timeout == 0:
|
||||
timeout = None
|
||||
|
||||
kwargs = {'timeout': timeout, 'fields': params, 'redirect': self.allow_redirect,
|
||||
'headers': self._get_headers(), 'preload_content': False}
|
||||
|
||||
if method in [self._MGET, self._MDELETE]:
|
||||
request_executor = self.http.request
|
||||
elif method in [self._MPUT, self._MPOST]:
|
||||
request_executor = self.http.request_encode_body
|
||||
kwargs['encode_multipart'] = False
|
||||
else:
|
||||
raise etcd.EtcdException('HTTP method {0} not supported'.format(method))
|
||||
|
||||
def api_execute(self, path, method, **kwargs):
|
||||
# Update machines_cache if previous attempt of update has failed
|
||||
if self._update_machines_cache:
|
||||
self._load_machines_cache()
|
||||
|
||||
response = False
|
||||
|
||||
self._update_machines_cache and self._load_machines_cache()
|
||||
try:
|
||||
while not response:
|
||||
response = self._do_http_request(request_executor, method, self._base_uri + path, **kwargs)
|
||||
|
||||
if response is False and not self._use_proxies:
|
||||
self._machines_cache = self.machines
|
||||
self._machines_cache.remove(self._base_uri)
|
||||
return self._handle_server_response(response)
|
||||
return super(Client, self).api_execute(path, method, **kwargs)
|
||||
except etcd.EtcdConnectionFailed:
|
||||
self._update_machines_cache = True
|
||||
raise
|
||||
@@ -113,6 +66,16 @@ class Client(etcd.Client):
|
||||
logger.exception('Can not resolve SRV for %s', host)
|
||||
return []
|
||||
|
||||
# try to workarond bug in python-etcd: https://github.com/jplana/python-etcd/issues/81
|
||||
def _result_from_response(self, response):
|
||||
try:
|
||||
response.data.decode('utf-8')
|
||||
except urllib3.exceptions.TimeoutError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise etcd.EtcdException('Unable to decode server response: %s' % e)
|
||||
return super(Client, self)._result_from_response(response)
|
||||
|
||||
def _get_machines_cache_from_srv(self, discovery_srv):
|
||||
"""Fetch list of etcd-cluster member by resolving _etcd-server._tcp. SRV record.
|
||||
This record should contain list of host and peer ports which could be used to run
|
||||
@@ -120,7 +83,7 @@ class Client(etcd.Client):
|
||||
|
||||
ret = []
|
||||
for host, port in self.get_srv_record(discovery_srv):
|
||||
url = '{0}://{1}:{2}/members'.format(self._protocol, host, port)
|
||||
url = '{}://{}:{}/members'.format(self._protocol, host, port)
|
||||
try:
|
||||
response = requests.get(url, timeout=5)
|
||||
if response.ok:
|
||||
@@ -138,10 +101,10 @@ class Client(etcd.Client):
|
||||
host, port = addr.split(':')
|
||||
try:
|
||||
for r in set(socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)):
|
||||
ret.append('{0}://{1}:{2}'.format(self._protocol, r[4][0], r[4][1]))
|
||||
ret.append('{}://{}:{}'.format(self._protocol, r[4][0], r[4][1]))
|
||||
except socket.error:
|
||||
logger.exception('Can not resolve %s', host)
|
||||
return list(set(ret)) if ret else ['{0}://{1}:{2}'.format(self._protocol, host, port)]
|
||||
return list(set(ret)) if ret else ['{}://{}:{}'.format(self._protocol, host, port)]
|
||||
|
||||
def _load_machines_cache(self):
|
||||
"""This method should fill up `_machines_cache` from scratch.
|
||||
@@ -169,9 +132,7 @@ class Client(etcd.Client):
|
||||
# After filling up initial list of machines_cache we should ask etcd-cluster about actual list
|
||||
self._base_uri = self._machines_cache.pop(0)
|
||||
self._machines_cache = self.machines
|
||||
|
||||
if self._base_uri in self._machines_cache:
|
||||
self._machines_cache.remove(self._base_uri)
|
||||
self._base_uri in self._machines_cache and self._machines_cache.remove(self._base_uri)
|
||||
|
||||
self._update_machines_cache = False
|
||||
|
||||
@@ -179,7 +140,7 @@ class Client(etcd.Client):
|
||||
def catch_etcd_errors(func):
|
||||
def wrapper(*args, **kwargs):
|
||||
try:
|
||||
return func(*args, **kwargs) is not None
|
||||
return not func(*args, **kwargs) is None
|
||||
except (RetryFailedError, etcd.EtcdException):
|
||||
return False
|
||||
except:
|
||||
@@ -204,8 +165,7 @@ class Etcd(AbstractDCS):
|
||||
def retry(self, *args, **kwargs):
|
||||
return self._retry.copy()(*args, **kwargs)
|
||||
|
||||
@staticmethod
|
||||
def get_etcd_client(config):
|
||||
def get_etcd_client(self, config):
|
||||
client = None
|
||||
while not client:
|
||||
try:
|
||||
@@ -225,25 +185,25 @@ class Etcd(AbstractDCS):
|
||||
nodes = {os.path.relpath(node.key, result.key): node for node in result.leaves}
|
||||
|
||||
# get initialize flag
|
||||
initialize = nodes.get(self._INITIALIZE)
|
||||
initialize = nodes.get(self._INITIALIZE, None)
|
||||
initialize = initialize and initialize.value
|
||||
|
||||
# get last leader operation
|
||||
last_leader_operation = nodes.get(self._LEADER_OPTIME)
|
||||
last_leader_operation = nodes.get(self._LEADER_OPTIME, None)
|
||||
last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation.value)
|
||||
|
||||
# get list of members
|
||||
members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
|
||||
|
||||
# get leader
|
||||
leader = nodes.get(self._LEADER)
|
||||
leader = nodes.get(self._LEADER, None)
|
||||
if leader:
|
||||
member = Member(-1, leader.value, None, {})
|
||||
member = ([m for m in members if m.name == leader.value] or [member])[0]
|
||||
leader = Leader(leader.modifiedIndex, leader.ttl, member)
|
||||
|
||||
# failover key
|
||||
failover = nodes.get(self._FAILOVER)
|
||||
failover = nodes.get(self._FAILOVER, None)
|
||||
if failover:
|
||||
failover = Failover.from_node(failover.modifiedIndex, failover.value)
|
||||
|
||||
@@ -308,7 +268,7 @@ class Etcd(AbstractDCS):
|
||||
# Synchronous work of all cluster members with etcd is less expensive
|
||||
# than reestablishing http connection every time from every replica.
|
||||
return True
|
||||
except etcd.EtcdWatchTimedOut:
|
||||
except urllib3.exceptions.TimeoutError:
|
||||
self.client.http.clear()
|
||||
return False
|
||||
except etcd.EtcdException:
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
from click import ClickException
|
||||
|
||||
|
||||
class PatroniException(Exception):
|
||||
|
||||
"""Parent class for all kind of exceptions related to selected distributed configuration store"""
|
||||
@@ -16,7 +13,7 @@ class PatroniException(Exception):
|
||||
return repr(self.value)
|
||||
|
||||
|
||||
class PatroniCtlException(ClickException):
|
||||
class PatroniCtlException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
|
||||
+157
-125
@@ -3,18 +3,15 @@ import logging
|
||||
import psycopg2
|
||||
import requests
|
||||
import sys
|
||||
import datetime
|
||||
import pytz
|
||||
|
||||
from multiprocessing.pool import ThreadPool
|
||||
from patroni.async_executor import AsyncExecutor
|
||||
from patroni.exceptions import DCSError, PostgresConnectionException
|
||||
from patroni.utils import sleep
|
||||
from multiprocessing.pool import ThreadPool
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Ha(object):
|
||||
class Ha:
|
||||
|
||||
def __init__(self, patroni):
|
||||
self.patroni = patroni
|
||||
@@ -22,13 +19,17 @@ class Ha(object):
|
||||
self.dcs = patroni.dcs
|
||||
self.cluster = None
|
||||
self.old_cluster = None
|
||||
self.recovering = False
|
||||
self._async_executor = AsyncExecutor()
|
||||
self.bdr = patroni.bdr
|
||||
self.use_bdr = self.bdr and\
|
||||
self.bdr.get('enable') and\
|
||||
(self.bdr.get('database') is not None)
|
||||
self.bdr_may_need_reinitialize = False
|
||||
|
||||
def load_cluster_from_dcs(self):
|
||||
cluster = self.dcs.get_cluster()
|
||||
|
||||
# We want to keep the state of cluster when it was healthy
|
||||
# We want to keep the state of cluster when it was healhy
|
||||
if not cluster.is_unlocked() or not self.old_cluster:
|
||||
self.old_cluster = cluster
|
||||
self.cluster = cluster
|
||||
@@ -50,7 +51,7 @@ class Ha(object):
|
||||
logger.info('Lock owner: %s; I am %s', lock_owner, self.state_handler.name)
|
||||
return lock_owner == self.state_handler.name
|
||||
|
||||
def touch_member(self):
|
||||
def touch_member(self, ttl=None):
|
||||
data = {
|
||||
'conn_url': self.state_handler.connection_string,
|
||||
'api_url': self.patroni.api.connection_string,
|
||||
@@ -63,31 +64,39 @@ class Ha(object):
|
||||
data['xlog_location'] = self.state_handler.xlog_position()
|
||||
except:
|
||||
pass
|
||||
self.dcs.touch_member(json.dumps(data, separators=(',', ':')))
|
||||
self.dcs.touch_member(json.dumps(data, separators=(',', ':')), ttl=ttl)
|
||||
|
||||
def clone(self, clone_member, clone_member_name="leader"):
|
||||
if self.state_handler.bootstrap(cluster_initialized=True, clone_member=clone_member):
|
||||
logger.info('bootstrapped from {0}'.format(clone_member_name)
|
||||
if clone_member else 'bootstrapped without leader')
|
||||
def copy_backup_from_leader(self, leader):
|
||||
if self.state_handler.bootstrap(leader):
|
||||
logger.info('bootstrapped from leader')
|
||||
else:
|
||||
self.state_handler.stop('immediate')
|
||||
self.state_handler.remove_data_directory()
|
||||
logger.error('failed to bootstrap from {0}'.format(clone_member_name)
|
||||
if clone_member else 'failed to bootstrap (without leader)')
|
||||
logger.error('failed to bootstrap from leader')
|
||||
|
||||
def bootstrap(self):
|
||||
def initialize_bdr(self, leader):
|
||||
if self.state_handler.initialize_bdr_database(leader):
|
||||
logger.info("bootstrapped from leader")
|
||||
else:
|
||||
self.state_handler.stop('immediate')
|
||||
self.state_handler.remove_data_directory()
|
||||
logger.error('failed to bootstrap from leader')
|
||||
|
||||
def bootstrap(self, bdr=False):
|
||||
if not self.cluster.is_unlocked(): # cluster already has leader
|
||||
clonefrom = self.patroni.clonefrom
|
||||
clone_member = self.cluster.get_member(clonefrom)\
|
||||
if self.cluster.has_member(clonefrom) else self.cluster.leader
|
||||
clone_member_name = 'leader' if clone_member == self.cluster.leader else 'replica \'{0}\''.format(clonefrom)
|
||||
self._async_executor.schedule('bootstrap from {0}'.format(clone_member_name))
|
||||
self._async_executor.run_async(self.clone, args=(clone_member, clone_member_name))
|
||||
return 'trying to bootstrap from {0}'.format(clone_member_name)
|
||||
self._async_executor.schedule('bootstrap from leader')
|
||||
if not bdr:
|
||||
self._async_executor.run_async(self.copy_backup_from_leader, args=(self.cluster.leader, ))
|
||||
else:
|
||||
self._async_executor.run_async(self.initialize_bdr, args=(self.cluster.leader, ))
|
||||
return 'trying to bootstrap from leader'
|
||||
elif not self.cluster.initialize and not self.patroni.nofailover: # no initialize key
|
||||
if self.dcs.initialize(create_new=True): # race for initialization
|
||||
try:
|
||||
self.state_handler.bootstrap()
|
||||
if not bdr:
|
||||
self.state_handler.bootstrap()
|
||||
else:
|
||||
self.state_handler.initialize_bdr_database()
|
||||
self.dcs.initialize(create_new=False, sysid=self.state_handler.sysid)
|
||||
except: # initdb or start failed
|
||||
# remove initialization key and give a chance to other members
|
||||
@@ -97,48 +106,64 @@ class Ha(object):
|
||||
self.state_handler.move_data_directory()
|
||||
raise
|
||||
self.dcs.take_leader()
|
||||
self.load_cluster_from_dcs()
|
||||
return 'initialized a new cluster'
|
||||
else:
|
||||
return 'failed to acquire initialize lock'
|
||||
else:
|
||||
if self.state_handler.can_create_replica_without_replication_connection():
|
||||
self._async_executor.run_async(self.clone, args=(None, ))
|
||||
return "trying to bootstrap without leader"
|
||||
return 'waiting for leader to bootstrap'
|
||||
|
||||
def bootstrap_bdr(self):
|
||||
return self.bootstrap(bdr=True)
|
||||
|
||||
def recover(self):
|
||||
has_lock = self.has_lock()
|
||||
|
||||
# try to see if we are the former master that crashed. If so - we likely need to run pg_rewind
|
||||
# in order to join the former standby being promoted.
|
||||
pg_controldata = self.state_handler.controldata()
|
||||
if (self.state_handler.role == 'master') and pg_controldata and\
|
||||
if not has_lock and pg_controldata and\
|
||||
pg_controldata.get('Database cluster state', '') == 'in production': # crashed master
|
||||
self.state_handler.require_rewind()
|
||||
self.recovering = True
|
||||
return self.follow("started as readonly because i had the session lock",
|
||||
"started as a secondary",
|
||||
refresh=True, recovery=True)
|
||||
|
||||
def follow(self, demote_reason, follow_reason, refresh=True, recovery=False):
|
||||
if refresh:
|
||||
self.load_cluster_from_dcs()
|
||||
# XXX: follow the leader calls stop, which might take quite some time.
|
||||
# perhaps we should run sync asynchronously
|
||||
# (we still need the exit code from follow_the_leader)
|
||||
ret = self.state_handler.follow_the_leader(None if has_lock else self.cluster.leader, recovery=True)
|
||||
if not ret:
|
||||
if not has_lock:
|
||||
return 'failed to start postgres'
|
||||
self.dcs.delete_leader()
|
||||
self.dcs.reset_cluster()
|
||||
return 'removed leader key after trying and failing to start postgres'
|
||||
if not has_lock:
|
||||
return 'started as a secondary'
|
||||
logger.info('started as readonly because i had the session lock')
|
||||
self.load_cluster_from_dcs()
|
||||
|
||||
if not recovery and self.state_handler.is_leader() or recovery and self.state_handler.role == 'master':
|
||||
ret = demote_reason
|
||||
else:
|
||||
ret = follow_reason
|
||||
def recover_bdr(self):
|
||||
if not self.bdr_may_need_reinitialize:
|
||||
self.state_handler.start_bdr()
|
||||
return "started as a member of BDR group"
|
||||
has_lock = self.has_lock()
|
||||
if self.cluster.is_unlocked(): # no leader yet, and we need to reinitialize from the leader
|
||||
return "waiting for another member of the BDR group"
|
||||
if has_lock:
|
||||
self.dcs.delete_leader()
|
||||
return "released the lock to another member of the BDR group"
|
||||
if self.state_handler.initialize_bdr_database(self.cluster.leader, empty=False):
|
||||
# we reinitialized this node, make sure the cluster is aware of it.
|
||||
self.touch_member()
|
||||
self.bdr_may_need_reinitialize = False
|
||||
return "reinitialized and started as a member of the BDR group"
|
||||
|
||||
# determine the node to follow. If replicatefrom tag is set,
|
||||
# try to follow the node mentioned there, otherwise, follow the leader.
|
||||
if self.patroni.replicatefrom:
|
||||
node_to_follow = [m for m in self.cluster.members if m.name == self.patroni.replicatefrom]
|
||||
node_to_follow = node_to_follow[0] if node_to_follow else self.cluster.leader
|
||||
else:
|
||||
node_to_follow = self.cluster.leader
|
||||
node_to_follow = None if node_to_follow and node_to_follow.name == self.state_handler.name else node_to_follow
|
||||
if not self.state_handler.check_recovery_conf(node_to_follow) or recovery:
|
||||
def follow_the_leader(self, demote_reason, follow_reason, refresh=True):
|
||||
refresh and self.load_cluster_from_dcs()
|
||||
ret = demote_reason if self.state_handler.is_leader() else follow_reason
|
||||
leader = self.cluster.leader
|
||||
leader = None if (leader and leader.name) == self.state_handler.name else leader
|
||||
if not self.state_handler.check_recovery_conf(leader):
|
||||
self._async_executor.schedule('changing primary_conninfo and restarting')
|
||||
self._async_executor.run_async(self.state_handler.follow, (node_to_follow, recovery))
|
||||
self._async_executor.run_async(self.state_handler.follow_the_leader, (leader, ))
|
||||
return ret
|
||||
|
||||
def enforce_master_role(self, message, promote_message):
|
||||
@@ -209,7 +234,7 @@ class Ha(object):
|
||||
ret = False
|
||||
members = [m for m in members if m.name != self.state_handler.name and not m.nofailover and m.api_url]
|
||||
if members:
|
||||
for member, reachable, _, _, tags in self.fetch_nodes_statuses(members):
|
||||
for member, reachable, in_recovery, xlog_location, tags in self.fetch_nodes_statuses(members):
|
||||
if reachable and not tags.get('nofailover', False):
|
||||
ret = True # TODO: check xlog_location
|
||||
elif not reachable:
|
||||
@@ -229,7 +254,7 @@ class Ha(object):
|
||||
# find specific node and check that it is healthy
|
||||
members = [m for m in self.cluster.members if m.name == failover.member]
|
||||
if members:
|
||||
member, reachable, _, _, tags = self.fetch_node_status(members[0])
|
||||
member, reachable, in_recovery, xlog_location, tags = self.fetch_node_status(members[0])
|
||||
if reachable and not tags.get('nofailover', False): # node is healthy
|
||||
logger.info('manual failover: to %s, i am %s', member.name, self.state_handler.name)
|
||||
return False
|
||||
@@ -279,40 +304,20 @@ class Ha(object):
|
||||
self.dcs.delete_leader()
|
||||
self.touch_member()
|
||||
self.dcs.reset_cluster()
|
||||
self.state_handler.follow(None)
|
||||
self.state_handler.follow_the_leader(None)
|
||||
|
||||
def process_manual_failover_from_leader(self):
|
||||
def process_manual_failover_from_leader(self, bdr=False):
|
||||
failover = self.cluster.failover
|
||||
|
||||
if failover.scheduled_at:
|
||||
# If the failover is in the far future, we shouldn't do anything and just return.
|
||||
# If the failover is in the past, we consider the value to be stale and we remove
|
||||
# the value.
|
||||
# If the value is close to now, we initiate the failover
|
||||
now = datetime.datetime.now(pytz.utc)
|
||||
try:
|
||||
delta = (failover.scheduled_at - now).total_seconds()
|
||||
|
||||
if delta > self.patroni.nap_time:
|
||||
logging.info('Awaiting failover at %s (in %.0f seconds)', failover.scheduled_at.isoformat(), delta)
|
||||
return
|
||||
elif delta < - int(self.patroni.nap_time * 1.5):
|
||||
logger.warning('Found a stale failover value, cleaning up: %s', failover.scheduled_at)
|
||||
self.dcs.manual_failover('', '', self.cluster.failover.index)
|
||||
return
|
||||
|
||||
# The value is very close to now
|
||||
sleep(max(delta, 0))
|
||||
logger.info('Manual scheduled failover at {}'.format(failover.scheduled_at.isoformat()))
|
||||
except TypeError:
|
||||
logger.warning('Incorrect value in of scheduled_at: %s', failover.scheduled_at)
|
||||
|
||||
if not failover.leader or failover.leader == self.state_handler.name:
|
||||
if not failover.member or failover.member != self.state_handler.name:
|
||||
members = [m for m in self.cluster.members if not failover.member or m.name == failover.member]
|
||||
if self.is_failover_possible(members): # check that there are healthy members
|
||||
self._async_executor.schedule('manual failover: demote')
|
||||
self._async_executor.run_async(self.demote)
|
||||
if not bdr:
|
||||
self._async_executor.schedule('manual failover: demote')
|
||||
self._async_executor.run_async(self.demote)
|
||||
else:
|
||||
self.dcs.delete_leader()
|
||||
self.dcs.reset_cluster()
|
||||
return 'manual failover: demoting myself'
|
||||
else:
|
||||
logger.warning('manual failover: no healthy members found, failover is not possible')
|
||||
@@ -325,6 +330,9 @@ class Ha(object):
|
||||
logger.info('Trying to clean up failover key')
|
||||
self.dcs.manual_failover('', '', self.cluster.failover.index)
|
||||
|
||||
def process_manual_failover_from_leader_bdr(self):
|
||||
self.process_manual_failover_from_leader(bdr=True)
|
||||
|
||||
def process_unhealthy_cluster(self):
|
||||
if self.is_healthiest_node():
|
||||
if self.acquire_lock():
|
||||
@@ -335,14 +343,14 @@ class Ha(object):
|
||||
return self.enforce_master_role('acquired session lock as a leader',
|
||||
'promoted self to leader by acquiring session lock')
|
||||
else:
|
||||
return self.follow('demoted self after trying and failing to obtain lock',
|
||||
'following new leader after trying and failing to obtain lock')
|
||||
return self.follow_the_leader('demoted self due after trying and failing to obtain lock',
|
||||
'following new leader after trying and failing to obtain lock')
|
||||
else:
|
||||
if self.patroni.nofailover:
|
||||
return self.follow('demoting self because I am not allowed to become master',
|
||||
'following a different leader because I am not allowed to promote')
|
||||
return self.follow('demoting self because i am not the healthiest node',
|
||||
'following a different leader because i am not the healthiest node')
|
||||
return self.follow_the_leader('demoting self because I am not allowed to become master',
|
||||
'following a different leader because I am not allowed to promote')
|
||||
return self.follow_the_leader('demoting self because i am not the healthiest node',
|
||||
'following a different leader because i am not the healthiest node')
|
||||
|
||||
def process_healthy_cluster(self):
|
||||
if self.has_lock():
|
||||
@@ -360,8 +368,33 @@ class Ha(object):
|
||||
self.load_cluster_from_dcs()
|
||||
else:
|
||||
logger.info('does not have lock')
|
||||
return self.follow('demoting self because i do not have the lock and i was a leader',
|
||||
'no action. i am a secondary and i am following a leader', False)
|
||||
return self.follow_the_leader('demoting self because i do not have the lock and i was a leader',
|
||||
'no action. i am a secondary and i am following a leader', False)
|
||||
|
||||
def process_unhealthy_cluster_bdr(self):
|
||||
if self.acquire_lock():
|
||||
if self.cluster.failover:
|
||||
logger.info('Cleaning up failover key after acquiring leader lock...')
|
||||
self.dcs.manual_failover('', '')
|
||||
self.dcs.get_cluster()
|
||||
return "acquired the session lock as a leader"
|
||||
else:
|
||||
return "lost the race to acquire the session lock"
|
||||
|
||||
def process_healthy_cluster_bdr(self):
|
||||
if self.has_lock():
|
||||
if self.cluster.failover:
|
||||
msg = self.process_manual_failover_from_leader_bdr()
|
||||
if msg is not None:
|
||||
return msg
|
||||
if self.update_lock():
|
||||
return "no action. I am the leader with the lock"
|
||||
else:
|
||||
logger.error("failed to update leader lock")
|
||||
self.load_cluster_from_dcs()
|
||||
else:
|
||||
logger.info("does not have lock")
|
||||
return "no action. I do not have the lock"
|
||||
|
||||
def schedule(self, action):
|
||||
with self._async_executor:
|
||||
@@ -389,7 +422,7 @@ class Ha(object):
|
||||
def reinitialize(self, cluster):
|
||||
self.state_handler.stop('immediate')
|
||||
self.state_handler.remove_data_directory()
|
||||
self.clone(cluster.leader)
|
||||
self.copy_backup_from_leader(cluster.leader)
|
||||
|
||||
def process_scheduled_action(self):
|
||||
if self.reinitialize_scheduled():
|
||||
@@ -414,26 +447,23 @@ class Ha(object):
|
||||
else:
|
||||
return self._async_executor.scheduled_action + ' in progress'
|
||||
|
||||
@staticmethod
|
||||
def sysid_valid(sysid):
|
||||
def sysid_valid(self, sysid):
|
||||
# sysid does tv_sec << 32, where tv_sec is the number of seconds sine 1970,
|
||||
# so even 1 << 32 would have 10 digits.
|
||||
return str(sysid) and len(str(sysid)) >= 10 and str(sysid).isdigit()
|
||||
|
||||
def post_recover(self):
|
||||
if not self.state_handler.is_running():
|
||||
if self.has_lock():
|
||||
self.dcs.delete_leader()
|
||||
self.dcs.reset_cluster()
|
||||
return 'removed leader key after trying and failing to start postgres'
|
||||
return 'failed to start postgres'
|
||||
return None
|
||||
|
||||
def _run_cycle(self):
|
||||
try:
|
||||
self.load_cluster_from_dcs()
|
||||
|
||||
self.touch_member()
|
||||
# if member key is missing - we may need to destory
|
||||
# and re-create the BDR database unless we are the first node in the cluster
|
||||
if self.use_bdr and len([m for m in self.cluster.members if m.name == self.state_handler.name]) == 0:
|
||||
self.bdr_may_need_reinitialize = True
|
||||
|
||||
# Create a member key with a very short expiration time if we may need to reinitialize this member
|
||||
# in order to avoid lossing the "may need to reinitialize status" in case Patroni is restarted
|
||||
self.touch_member(ttl=None if not self.bdr_may_need_reinitialize else self.bdr.get('min_ttl', 5))
|
||||
|
||||
# cluster has leader key but not initialize key
|
||||
if not self.cluster.is_unlocked() and not self.sysid_valid(self.cluster.initialize) and self.has_lock():
|
||||
@@ -442,13 +472,6 @@ class Ha(object):
|
||||
if self._async_executor.busy:
|
||||
return self.handle_long_action_in_progress()
|
||||
|
||||
# we've got here, so any async action has finished. Check if we tried to recover and failed
|
||||
if self.recovering:
|
||||
self.recovering = False
|
||||
msg = self.post_recover()
|
||||
if msg is not None:
|
||||
return msg
|
||||
|
||||
# currently it can trigger only reinitialize
|
||||
msg = self.process_scheduled_action()
|
||||
if msg is not None:
|
||||
@@ -456,41 +479,50 @@ class Ha(object):
|
||||
|
||||
# is data directory empty?
|
||||
if self.state_handler.data_directory_empty():
|
||||
return self.bootstrap() # new node
|
||||
if self.use_bdr:
|
||||
# member key was missing, but it's a new database - no need to reinitialize
|
||||
self.bdr_may_need_reinitialize = False
|
||||
self.touch_member()
|
||||
return self.bootstrap(self.use_bdr) # new node
|
||||
# "bootstrap", but data directory is not empty
|
||||
elif not self.sysid_valid(self.cluster.initialize) and self.cluster.is_unlocked():
|
||||
elif not self.sysid_valid(self.cluster.initialize) and self.cluster.is_unlocked() and \
|
||||
not self.bdr_may_need_reinitialize:
|
||||
self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=self.state_handler.sysid)
|
||||
else:
|
||||
# check if we are allowed to join
|
||||
if self.sysid_valid(self.cluster.initialize) and self.cluster.initialize != self.state_handler.sysid:
|
||||
logger.fatal("system ID mismatch, node %s belongs to a different cluster: %s != %s",
|
||||
self.state_handler.name, self.cluster.initialize, self.state_handler.sysid)
|
||||
if not self.use_bdr and \
|
||||
self.sysid_valid(self.cluster.initialize) and self.cluster.initialize != self.state_handler.sysid:
|
||||
logger.fatal("system ID mismatch, node {0} belongs to a different cluster".
|
||||
format(self.state_handler.name))
|
||||
sys.exit(1)
|
||||
|
||||
# try to start dead postgres
|
||||
if not self.state_handler.is_healthy():
|
||||
msg = self.recover()
|
||||
if msg is not None:
|
||||
return msg
|
||||
if self.use_bdr:
|
||||
if self.bdr_may_need_reinitialize or not self.state_handler.is_healthy():
|
||||
msg = self.recover_bdr()
|
||||
if msg is not None:
|
||||
return msg
|
||||
else:
|
||||
if not self.state_handler.is_healthy():
|
||||
msg = self.recover()
|
||||
if msg is not None:
|
||||
return msg
|
||||
|
||||
try:
|
||||
if self.cluster.is_unlocked():
|
||||
return self.process_unhealthy_cluster()
|
||||
return self.process_unhealthy_cluster() if not self.use_bdr else self.process_unhealthy_cluster_bdr()
|
||||
else:
|
||||
return self.process_healthy_cluster()
|
||||
return self.process_healthy_cluster() if not self.use_bdr else self.process_healthy_cluster_bdr()
|
||||
finally:
|
||||
# we might not have a valid PostgreSQL connection here if another thread
|
||||
# stops PostgreSQL, therefore, we only reload replication slots if no
|
||||
# asynchronous processes are running (should be always the case for the master)
|
||||
if not self._async_executor.busy:
|
||||
if not self.use_bdr:
|
||||
self.state_handler.sync_replication_slots(self.cluster)
|
||||
except DCSError:
|
||||
logger.error('Error communicating with DCS')
|
||||
if self.state_handler.is_running() and self.state_handler.is_leader():
|
||||
if self.state_handler.is_running() and self.state_handler.is_leader() and not self.use_bdr:
|
||||
self.demote(delete_leader=False)
|
||||
return 'demoted self because DCS is not accessible and i was a leader'
|
||||
except (psycopg2.Error, PostgresConnectionException):
|
||||
logger.exception('Error communicating with PostgreSQL. Will try again later')
|
||||
logger.exception('Error communicating with Postgresql. Will try again later')
|
||||
|
||||
def run_cycle(self):
|
||||
with self._async_executor:
|
||||
|
||||
+248
-205
@@ -1,6 +1,7 @@
|
||||
import logging
|
||||
import os
|
||||
import psycopg2
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -39,10 +40,11 @@ def parseurl(url):
|
||||
return ret
|
||||
|
||||
|
||||
class Postgresql(object):
|
||||
class Postgresql:
|
||||
|
||||
def __init__(self, config):
|
||||
def __init__(self, config, config_bdr={}):
|
||||
self.config = config
|
||||
self.bdr = config_bdr
|
||||
self.name = config['name']
|
||||
self.server_parameters = config.get('parameters', {})
|
||||
self.scope = config['scope']
|
||||
@@ -52,7 +54,7 @@ class Postgresql(object):
|
||||
self.superuser = config['superuser']
|
||||
self.admin = config['admin']
|
||||
self.initdb_options = config.get('initdb', [])
|
||||
self.pgpass = config.get('pgpass') or os.path.join(os.path.expanduser('~'), 'pgpass')
|
||||
self.pgpass = config.get('pgpass', None) or os.path.join(os.path.expanduser('~'), 'pgpass')
|
||||
self.pg_rewind = config.get('pg_rewind', {})
|
||||
self.callback = config.get('callbacks', {})
|
||||
self.use_slots = config.get('use_slots', True)
|
||||
@@ -61,17 +63,18 @@ class Postgresql(object):
|
||||
self.configuration_to_save = (os.path.join(self.data_dir, 'pg_hba.conf'),
|
||||
os.path.join(self.data_dir, 'postgresql.conf'))
|
||||
self.postmaster_pid = os.path.join(self.data_dir, 'postmaster.pid')
|
||||
self.trigger_file = config.get('recovery_conf', {}).get('trigger_file') or 'promote'
|
||||
self.trigger_file = config.get('recovery_conf', {}).get('trigger_file', None) or 'promote'
|
||||
self.trigger_file = os.path.abspath(os.path.join(self.data_dir, self.trigger_file))
|
||||
|
||||
self._pg_ctl = ['pg_ctl', '-w', '-D', self.data_dir]
|
||||
|
||||
self.local_address = self.get_local_address()
|
||||
connect_address = config.get('connect_address') or self.local_address
|
||||
connect_address = config.get('connect_address', None) or self.local_address
|
||||
self.connection_string = 'postgres://{username}:{password}@{connect_address}/postgres'.format(
|
||||
connect_address=connect_address, **self.replication)
|
||||
|
||||
self._connection = None
|
||||
self._connection_db = None
|
||||
self._cursor_holder = None
|
||||
self._need_rewind = False
|
||||
self._sysid = None
|
||||
@@ -128,37 +131,32 @@ class Postgresql(object):
|
||||
break
|
||||
return local_address + ':' + self.port
|
||||
|
||||
@property
|
||||
def _connect_kwargs(self):
|
||||
r = parseurl('postgres://{0}/postgres'.format(self.local_address))
|
||||
if 'username' in self.superuser:
|
||||
r['user'] = self.superuser['username']
|
||||
if 'password' in self.superuser:
|
||||
r['password'] = self.superuser['password']
|
||||
return r
|
||||
|
||||
def connection(self):
|
||||
def connection(self, dbname):
|
||||
if not self._connection or self._connection.closed != 0:
|
||||
self._connection = psycopg2.connect(**self._connect_kwargs)
|
||||
r = parseurl('postgres://{0}/{1}'.format(self.local_address, dbname))
|
||||
self._connection = psycopg2.connect(**r)
|
||||
self._connection_db = dbname
|
||||
self._connection.autocommit = True
|
||||
self.server_version = self._connection.server_version
|
||||
return self._connection
|
||||
|
||||
def _cursor(self):
|
||||
def _cursor(self, dbname='postgres'):
|
||||
if self._connection_db != dbname:
|
||||
self.close_connection()
|
||||
if not self._cursor_holder or self._cursor_holder.closed or self._cursor_holder.connection.closed != 0:
|
||||
logger.info("establishing a new patroni connection to the postgres cluster")
|
||||
self._cursor_holder = self.connection().cursor()
|
||||
logger.info("established a new patroni connection to the {0} cluster".format(dbname))
|
||||
self._cursor_holder = self.connection(dbname).cursor()
|
||||
return self._cursor_holder
|
||||
|
||||
def close_connection(self):
|
||||
if self._cursor_holder and self._cursor_holder.connection and self._cursor_holder.connection.closed == 0:
|
||||
self._cursor_holder.connection.close()
|
||||
logger.info("closed patroni connection to the postgresql cluster")
|
||||
logger.info("closed patroni connection to the {0} cluster".format(self._connection_db))
|
||||
self._connection_db = None
|
||||
|
||||
def _query(self, sql, *params):
|
||||
def _query(self, sql, dbname, *params):
|
||||
cursor = None
|
||||
try:
|
||||
cursor = self._cursor()
|
||||
cursor = self._cursor(dbname)
|
||||
cursor.execute(sql, params)
|
||||
return cursor
|
||||
except psycopg2.Error as e:
|
||||
@@ -168,9 +166,10 @@ class Postgresql(object):
|
||||
raise RetryFailedError('cluster is being restarted')
|
||||
raise PostgresConnectionException('connection problems')
|
||||
|
||||
def query(self, sql, *params):
|
||||
def query(self, sql, *params, **kwargs):
|
||||
dbname = kwargs.get('dbname', 'postgres')
|
||||
try:
|
||||
return self.retry(self._query, sql, *params)
|
||||
return self.retry(self._query, sql, dbname, *params)
|
||||
except RetryFailedError as e:
|
||||
raise PostgresConnectionException(str(e))
|
||||
|
||||
@@ -180,36 +179,32 @@ class Postgresql(object):
|
||||
@staticmethod
|
||||
def initdb_allowed_option(name):
|
||||
if name in ['pgdata', 'nosync', 'pwfile', 'sync-only']:
|
||||
raise Exception('{0} option for initdb is not allowed'.format(name))
|
||||
raise Exception('{} option for initdb is not allowed'.format(name))
|
||||
return True
|
||||
|
||||
def get_initdb_options(self):
|
||||
options = []
|
||||
for o in self.initdb_options:
|
||||
if isinstance(o, string_types) and self.initdb_allowed_option(o):
|
||||
options.append('--{0}'.format(o))
|
||||
options.append('--{}'.format(o))
|
||||
elif isinstance(o, dict):
|
||||
keys = list(o.keys())
|
||||
if len(keys) != 1 or not isinstance(keys[0], string_types) or not self.initdb_allowed_option(keys[0]):
|
||||
raise Exception('Invalid option: {0}'.format(o))
|
||||
options.append('--{0}={1}'.format(keys[0], o[keys[0]]))
|
||||
raise Exception('Invalid option: {}'.format(o))
|
||||
options.append('--{}={}'.format(keys[0], o[keys[0]]))
|
||||
else:
|
||||
raise Exception('Unknown type of initdb option: {0}'.format(o))
|
||||
raise Exception('Unknown type of initdb option: {}'.format(o))
|
||||
return options
|
||||
|
||||
def initialize(self):
|
||||
self.set_state('initalizing new cluster')
|
||||
options = self.get_initdb_options()
|
||||
pwfile = None
|
||||
|
||||
if self.superuser:
|
||||
if 'username' in self.superuser:
|
||||
options.append('--username={0}'.format(self.superuser['username']))
|
||||
if 'password' in self.superuser:
|
||||
(fd, pwfile) = tempfile.mkstemp()
|
||||
os.write(fd, self.superuser['password'].encode('utf-8'))
|
||||
os.close(fd)
|
||||
options.append('--pwfile={0}'.format(pwfile))
|
||||
if self.superuser and 'username' not in self.superuser and 'password' in self.superuser:
|
||||
(fd, pwfile) = tempfile.mkstemp()
|
||||
os.write(fd, self.superuser['password'].encode())
|
||||
os.close(fd)
|
||||
options.append('--pwfile={}'.format(pwfile))
|
||||
|
||||
ret = subprocess.call(self._pg_ctl + ['initdb'] + (['-o', ' '.join(options)] if options else [])) == 0
|
||||
if pwfile:
|
||||
@@ -220,9 +215,83 @@ class Postgresql(object):
|
||||
self.set_state('initdb failed')
|
||||
return ret
|
||||
|
||||
def initialize_bdr_database(self, leader=None, empty=True):
|
||||
""" initialize BDR database. If empty = False, reinitialize an existing one.
|
||||
If leader is None, create a new group, otherwise, join an exising one.
|
||||
non-empty database with an empty leader is not supported
|
||||
"""
|
||||
if not empty and not leader:
|
||||
ret = False
|
||||
else:
|
||||
try:
|
||||
ret = (self.initialize() and self.start_bdr()) if empty else\
|
||||
(self.drop_bdr_database() and self.start_bdr())
|
||||
except:
|
||||
logger.exception("exception")
|
||||
ret = False
|
||||
if ret:
|
||||
# convert all connection URLs to name=value strings
|
||||
leader_dsn = self.primary_conninfo(leader.conn_url, include_db=self.bdr['database']) if leader else None
|
||||
self_dsn = self.primary_conninfo(self.connection_string, include_db=self.bdr['database'])
|
||||
try:
|
||||
self.create_replication_user(superuser=True)
|
||||
ret = self.create_bdr_database() and (self.join_bdr_group(self_dsn, leader_dsn) if leader
|
||||
else self.create_bdr_group(self_dsn))
|
||||
except:
|
||||
logger.exception("exception")
|
||||
ret = False
|
||||
if not ret and not leader:
|
||||
raise Exception("Could not bootstrap initial PostgreSQL BDR node")
|
||||
return ret
|
||||
|
||||
def start_bdr(self):
|
||||
logger.info("starting new BDR instance {0}".format(self.name))
|
||||
return self.start(bdr=True)
|
||||
|
||||
def drop_bdr_database(self):
|
||||
logger.info("Dropping BDR database for instance {0}".format(self.name))
|
||||
# stop the database if it's running in order to turn off bdr
|
||||
if self.is_running():
|
||||
self.stop()
|
||||
ret = self.start(bdr=False)
|
||||
if ret:
|
||||
self.query("DROP DATABASE IF EXISTS {0}".format(self.bdr['database']))
|
||||
return ret and self.restart(bdr=True)
|
||||
|
||||
def create_bdr_database(self):
|
||||
logger.info("Creating BDR database and extensions for instance {0}".format(self.name))
|
||||
self.query("CREATE DATABASE {0}".format(self.bdr['database']))
|
||||
self.query("CREATE EXTENSION IF NOT EXISTS btree_gist", dbname=self.bdr['database'])
|
||||
self.query("CREATE EXTENSION IF NOT EXISTS bdr", dbname=self.bdr['database'])
|
||||
return True
|
||||
|
||||
def join_bdr_group(self, self_dsn, join_dsn):
|
||||
logger.info("Joining BDR group {0}, host {1}, dsn {2}".format(join_dsn, self.name, self_dsn))
|
||||
self.query("SELECT bdr.bdr_group_join(local_node_name:=%s, join_using_dsn:=%s, node_external_dsn:=%s)",
|
||||
self.name, join_dsn, self_dsn, dbname=self.bdr['database'])
|
||||
return True
|
||||
|
||||
def create_bdr_group(self, self_dsn):
|
||||
logger.info("Creating new BDR group {0}, host {1}".format(self_dsn, self.name))
|
||||
self.query("SELECT bdr.bdr_group_create(local_node_name:=%s, node_external_dsn:=%s)",
|
||||
self.name, self_dsn, dbname=self.bdr['database'])
|
||||
return True
|
||||
|
||||
def remove_bdr_nodes(self, nodes):
|
||||
logger.info("BDR node {0}, removing nodes {1}".format(self.name, nodes))
|
||||
self.query("SELECT bdr.part_by_node_names(%s)", nodes, dbname=self.bdr['database'])
|
||||
|
||||
def remove_bdr_disconnected_members(self, cluster):
|
||||
# get all members
|
||||
result = self.query("SELECT node_name FROM bdr.bdr_nodes WHERE node_status = 'r'")
|
||||
if result:
|
||||
nodes_db = set([node[0] for node in result.fetchall()])
|
||||
nodes_etcd = set([member.name for member in self.members])
|
||||
nodes_remove = nodes_db.difference(nodes_etcd)
|
||||
self.remove_bdr_nodes(list(nodes_remove))
|
||||
|
||||
def delete_trigger_file(self):
|
||||
if os.path.exists(self.trigger_file):
|
||||
os.unlink(self.trigger_file)
|
||||
os.path.exists(self.trigger_file) and os.unlink(self.trigger_file)
|
||||
|
||||
def write_pgpass(self, record):
|
||||
with open(self.pgpass, 'w') as f:
|
||||
@@ -233,13 +302,13 @@ class Postgresql(object):
|
||||
env['PGPASSFILE'] = self.pgpass
|
||||
return env
|
||||
|
||||
def sync_replica(self, clone_member):
|
||||
# add the credentials to connect to the replica origin to pgpass.
|
||||
env = self.write_pgpass(parseurl(clone_member.conn_url)) if clone_member else os.environ.copy()
|
||||
if self.create_replica(clone_member, env) == 0:
|
||||
self.delete_trigger_file()
|
||||
return True
|
||||
return False
|
||||
def sync_from_leader(self, leader):
|
||||
r = parseurl(leader.conn_url)
|
||||
|
||||
env = self.write_pgpass(r)
|
||||
ret = self.create_replica(leader, env) == 0
|
||||
ret and self.delete_trigger_file()
|
||||
return ret
|
||||
|
||||
@staticmethod
|
||||
def build_connstring(conn):
|
||||
@@ -247,37 +316,22 @@ class Postgresql(object):
|
||||
>>> Postgresql.build_connstring({'host': '127.0.0.1', 'port': '5432'}) == 'host=127.0.0.1 port=5432'
|
||||
True
|
||||
"""
|
||||
return ' '.join('{0}={1}'.format(param, val) for param, val in sorted(conn.items()))
|
||||
return ' '.join('{}={}'.format(param, val) for param, val in sorted(conn.items()))
|
||||
|
||||
def replica_method_can_work_without_replication_connection(self, method):
|
||||
return method != 'basebackup' and self.config and self.config.get(method, {}).get('no_master')
|
||||
|
||||
def can_create_replica_without_replication_connection(self):
|
||||
""" go through the replication methods to see if there are ones
|
||||
that does not require a working replication connection.
|
||||
"""
|
||||
replica_methods = self.config.get('create_replica_method', [])
|
||||
return any(self.replica_method_can_work_without_replication_connection(replica_method)
|
||||
for replica_method in replica_methods)
|
||||
|
||||
def create_replica(self, clone_member, env):
|
||||
def create_replica(self, leader, env):
|
||||
# create the replica according to the replica_method
|
||||
# defined by the user. this is a list, so we need to
|
||||
# loop through all methods the user supplies
|
||||
connstring = clone_member.conn_url if clone_member else ""
|
||||
connstring = leader.conn_url
|
||||
# get list of replica methods from config.
|
||||
# If there is no configuration key, or no value is specified, use basebackup
|
||||
replica_methods = self.config.get('create_replica_method') or ['basebackup']
|
||||
# if we don't have any source, leave only replica methods that work without it
|
||||
replica_methods = \
|
||||
[r for r in replica_methods if self.replica_method_can_work_without_replication_connection(r)]\
|
||||
if not clone_member else replica_methods
|
||||
# go through them in priority order
|
||||
ret = 1
|
||||
for replica_method in replica_methods:
|
||||
# if the method is basebackup, then use the built-in
|
||||
if replica_method == "basebackup":
|
||||
ret = self.basebackup(clone_member, env)
|
||||
ret = self.basebackup(leader, env)
|
||||
if ret == 0:
|
||||
logger.info("replica has been created using basebackup")
|
||||
# if basebackup succeeds, exit with success
|
||||
@@ -324,7 +378,7 @@ class Postgresql(object):
|
||||
cmd = self.callback[cb_name]
|
||||
try:
|
||||
subprocess.Popen(shlex.split(cmd) + [cb_name, self.role, self.scope])
|
||||
except OSError:
|
||||
except:
|
||||
logger.exception('callback %s %s %s %s failed', cmd, cb_name, self.role, self.scope)
|
||||
return False
|
||||
return True
|
||||
@@ -347,12 +401,15 @@ class Postgresql(object):
|
||||
with self._state_lock:
|
||||
self._state = value
|
||||
|
||||
def start(self, block_callbacks=False):
|
||||
def start(self, block_callbacks=False, bdr=False):
|
||||
if self.is_running():
|
||||
logger.error('Cannot start PostgreSQL because one is already running.')
|
||||
return True
|
||||
|
||||
self.set_role('replica' if os.path.exists(self.recovery_conf) else 'master')
|
||||
if not bdr:
|
||||
self.set_role('replica' if os.path.exists(self.recovery_conf) else 'master')
|
||||
else:
|
||||
self.set_role('master')
|
||||
if os.path.exists(self.postmaster_pid):
|
||||
os.remove(self.postmaster_pid)
|
||||
logger.info('Removed %s', self.postmaster_pid)
|
||||
@@ -360,32 +417,27 @@ class Postgresql(object):
|
||||
if not block_callbacks:
|
||||
self.set_state('starting')
|
||||
|
||||
env = os.environ.copy()
|
||||
if 'username' in self.superuser:
|
||||
env['PGUSER'] = self.superuser['username']
|
||||
ret = subprocess.call(self._pg_ctl + ['start', '-o', self.server_options()], env=env, preexec_fn=os.setsid) == 0
|
||||
ret = subprocess.call(self._pg_ctl + ['start', '-o', self.server_options(bdr)]) == 0
|
||||
|
||||
self.set_state('running' if ret else 'start failed')
|
||||
|
||||
self.schedule_load_slots = ret and self.use_slots
|
||||
self.save_configuration_files()
|
||||
if not bdr:
|
||||
self.schedule_load_slots = ret and self.use_slots
|
||||
self.save_configuration_files()
|
||||
# block_callbacks is used during restart to avoid
|
||||
# running start/stop callbacks in addition to restart ones
|
||||
if ret and not block_callbacks:
|
||||
self.call_nowait(ACTION_ON_START)
|
||||
ret and not block_callbacks and self.call_nowait(ACTION_ON_START)
|
||||
return ret
|
||||
|
||||
def checkpoint(self, connect_kwargs=None):
|
||||
connect_kwargs = connect_kwargs or self._connect_kwargs
|
||||
for p in ['connect_timeout', 'options']:
|
||||
connect_kwargs.pop(p, None)
|
||||
def checkpoint(self, connstring=None):
|
||||
try:
|
||||
with psycopg2.connect(**connect_kwargs) as conn:
|
||||
connstring = connstring or 'postgres://{}/postgres'.format(self.local_address)
|
||||
with psycopg2.connect(connstring) as conn:
|
||||
conn.autocommit = True
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SET statement_timeout = 0")
|
||||
cur.execute('CHECKPOINT')
|
||||
except psycopg2.Error:
|
||||
except:
|
||||
logging.exception('Exception during CHECKPOINT')
|
||||
|
||||
def stop(self, mode='fast', block_callbacks=False):
|
||||
@@ -417,23 +469,31 @@ class Postgresql(object):
|
||||
|
||||
def reload(self):
|
||||
ret = subprocess.call(self._pg_ctl + ['reload']) == 0
|
||||
if ret:
|
||||
self.call_nowait(ACTION_ON_RELOAD)
|
||||
ret and self.call_nowait(ACTION_ON_RELOAD)
|
||||
return ret
|
||||
|
||||
def restart(self):
|
||||
def restart(self, bdr=False):
|
||||
self.set_state('restarting')
|
||||
ret = self.stop(block_callbacks=True) and self.start(block_callbacks=True)
|
||||
ret = self.stop(block_callbacks=True) and self.start(block_callbacks=True, bdr=bdr)
|
||||
if ret:
|
||||
self.call_nowait(ACTION_ON_RESTART)
|
||||
else:
|
||||
self.set_state('restart failed ({0})'.format(self.state))
|
||||
self.set_state('restart failed ({})'.format(self.state))
|
||||
return ret
|
||||
|
||||
def server_options(self):
|
||||
options = "--listen_addresses='{0}' --port={1}".format(self.listen_addresses, self.port)
|
||||
def server_options(self, bdr=False):
|
||||
bdr_set = False
|
||||
options = "--listen_addresses='{}' --port={}".format(self.listen_addresses, self.port)
|
||||
for setting, value in self.server_parameters.items():
|
||||
options += " --{0}='{1}'".format(setting, value)
|
||||
if setting == 'shared_preload_libraries':
|
||||
if not bdr and 'bdr' in value:
|
||||
value = ','.join([x for x in value.split(',') if x != 'bdr'])
|
||||
elif bdr and 'bdr' not in value:
|
||||
bdr_set = True
|
||||
value = ','.join([x for x in value.split(',')]+['bdr'])
|
||||
options += " --{}='{}'".format(setting, value)
|
||||
if bdr and not bdr_set:
|
||||
options += " --shared_preload_libraries='bdr'"
|
||||
return options
|
||||
|
||||
def is_healthy(self):
|
||||
@@ -443,7 +503,8 @@ class Postgresql(object):
|
||||
return True
|
||||
|
||||
def check_replication_lag(self, last_leader_operation):
|
||||
return (last_leader_operation or 0) - self.xlog_position() <= self.config.get('maximum_lag_on_failover', 0)
|
||||
return (last_leader_operation if last_leader_operation else 0) - self.xlog_position() <=\
|
||||
self.config.get('maximum_lag_on_failover', 0)
|
||||
|
||||
def write_pg_hba(self):
|
||||
with open(os.path.join(self.data_dir, 'pg_hba.conf'), 'a') as f:
|
||||
@@ -454,9 +515,10 @@ class Postgresql(object):
|
||||
f.write(line + '\n')
|
||||
|
||||
@staticmethod
|
||||
def primary_conninfo(leader_url):
|
||||
def primary_conninfo(leader_url, include_db=None):
|
||||
r = parseurl(leader_url)
|
||||
return 'user={user} password={password} host={host} port={port} sslmode=prefer sslcompression=1'.format(**r)
|
||||
ret = 'user={user} password={password} host={host} port={port} sslmode=prefer sslcompression=1'.format(**r)
|
||||
return ret if not include_db else ret + ' dbname={0}'.format(include_db)
|
||||
|
||||
def check_recovery_conf(self, leader):
|
||||
if not os.path.isfile(self.recovery_conf):
|
||||
@@ -470,34 +532,33 @@ class Postgresql(object):
|
||||
return pattern and (pattern in line)
|
||||
return not pattern
|
||||
|
||||
def write_recovery_conf(self, leader, bootstrap=False):
|
||||
def write_recovery_conf(self, leader):
|
||||
with open(self.recovery_conf, 'w') as f:
|
||||
f.write("""standby_mode = 'on'
|
||||
recovery_target_timeline = 'latest'
|
||||
""")
|
||||
if leader and leader.conn_url:
|
||||
f.write("""primary_conninfo = '{0}'\n""".format(self.primary_conninfo(leader.conn_url)))
|
||||
f.write("""primary_conninfo = '{}'\n""".format(self.primary_conninfo(leader.conn_url)))
|
||||
if self.use_slots:
|
||||
f.write("""primary_slot_name = '{0}'\n""".format(self.name))
|
||||
if (leader and leader.conn_url) or bootstrap:
|
||||
f.write("""primary_slot_name = '{}'\n""".format(self.name))
|
||||
for name, value in self.config.get('recovery_conf', {}).items():
|
||||
f.write("{0} = '{1}'\n".format(name, value))
|
||||
f.write("{} = '{}'\n".format(name, value))
|
||||
|
||||
def rewind(self, leader):
|
||||
# prepare pg_rewind connection
|
||||
r = parseurl(leader.conn_url)
|
||||
r.update(self.pg_rewind)
|
||||
r['user'] = r.pop('username')
|
||||
r['user'] = r['username']
|
||||
env = self.write_pgpass(r)
|
||||
pc = "user={user} host={host} port={port} dbname=postgres sslmode=prefer sslcompression=1".format(**r)
|
||||
# first run a checkpoint on a promoted master in order
|
||||
# to make it store the new timeline ([email protected])
|
||||
self.checkpoint(r)
|
||||
logger.info("running pg_rewind from %s", pc)
|
||||
self.checkpoint(pc)
|
||||
logger.info("running pg_rewind from {}".format(pc))
|
||||
pg_rewind = ['pg_rewind', '-D', self.data_dir, '--source-server', pc]
|
||||
try:
|
||||
ret = subprocess.call(pg_rewind, env=env) == 0
|
||||
except OSError:
|
||||
ret = (subprocess.call(pg_rewind, env=env) == 0)
|
||||
except:
|
||||
ret = False
|
||||
if ret:
|
||||
self.write_recovery_conf(leader)
|
||||
@@ -509,11 +570,12 @@ recovery_target_timeline = 'latest'
|
||||
try:
|
||||
data = subprocess.check_output(['pg_controldata', self.data_dir])
|
||||
if data:
|
||||
data = data.decode('utf-8').splitlines()
|
||||
data = data.decode().splitlines()
|
||||
result = {l.split(':')[0].replace('Current ', '', 1): l.split(':')[1].strip() for l in data if l}
|
||||
except subprocess.CalledProcessError:
|
||||
logger.exception("Error when calling pg_controldata")
|
||||
return result
|
||||
finally:
|
||||
return result
|
||||
|
||||
def read_postmaster_opts(self):
|
||||
""" returns the list of option names/values from postgres.opts, Empty dict if read failed or no file """
|
||||
@@ -529,26 +591,26 @@ recovery_target_timeline = 'latest'
|
||||
result[name] = val
|
||||
except IOError:
|
||||
logger.exception('Error when reading postmaster.opts')
|
||||
return result
|
||||
finally:
|
||||
return result
|
||||
|
||||
def single_user_mode(self, command=None, options=None):
|
||||
def single_user_mode(self, command=None, options={}):
|
||||
""" run a given command in a single-user mode. If the command is empty - then just start and stop """
|
||||
cmd = ['postgres', '--single', '-D', self.data_dir]
|
||||
for opt, val in sorted((options or {}).items()):
|
||||
cmd.extend(['-c', '{0}={1}'.format(opt, val)])
|
||||
for opt in sorted(options):
|
||||
cmd.extend(['-c', '{0}={1}'.format(opt, options[opt])])
|
||||
# need a database name to connect
|
||||
cmd.append('postgres')
|
||||
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT)
|
||||
if p:
|
||||
if command:
|
||||
p.communicate('{0}\n'.format(command))
|
||||
command and p.communicate('{}\n'.format(command))
|
||||
p.stdin.close()
|
||||
return p.wait()
|
||||
return 1
|
||||
|
||||
def cleanup_archive_status(self):
|
||||
status_dir = os.path.join(self.data_dir, 'pg_xlog', 'archive_status')
|
||||
try:
|
||||
if os.path.isdir(status_dir):
|
||||
for f in os.listdir(status_dir):
|
||||
path = os.path.join(status_dir, f)
|
||||
try:
|
||||
@@ -556,51 +618,50 @@ recovery_target_timeline = 'latest'
|
||||
os.unlink(path)
|
||||
elif os.path.isfile(path):
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
logger.exception("Unable to remove %s", path)
|
||||
except OSError:
|
||||
logger.exception("Unable to list %s", status_dir)
|
||||
except:
|
||||
logger.exception("Unable to remove {}".format(path))
|
||||
|
||||
def follow(self, leader, recovery=False):
|
||||
if self.check_recovery_conf(leader) and not recovery:
|
||||
def follow_the_leader(self, leader, recovery=False):
|
||||
if not self.check_recovery_conf(leader) or recovery:
|
||||
change_role = (self.role == 'master')
|
||||
|
||||
self._need_rewind = (self._need_rewind or change_role) and self.can_rewind
|
||||
if self._need_rewind:
|
||||
logger.info("set the rewind flag after demote")
|
||||
self.write_recovery_conf(leader)
|
||||
if not leader or not self._need_rewind: # do not rewind until the leader becomes available
|
||||
ret = self.restart()
|
||||
else: # we have a leader and need to rewind
|
||||
if self.is_running():
|
||||
self.stop()
|
||||
# at present, pg_rewind only runs when the cluster is shut down cleanly
|
||||
# and not shutdown in recovery. We have to remove the recovery.conf if present
|
||||
# and start/shutdown in a single user mode to emulate this.
|
||||
# XXX: if recovery.conf is linked, it will be written anew as a normal file.
|
||||
if os.path.islink(self.recovery_conf):
|
||||
os.unlink(self.recovery_conf)
|
||||
else:
|
||||
os.remove(self.recovery_conf)
|
||||
# Archived segments might be useful to pg_rewind,
|
||||
# clean the flags that tell we should remove them.
|
||||
self.cleanup_archive_status()
|
||||
# Start in a single user mode and stop to produce a clean shutdown
|
||||
opts = self.read_postmaster_opts()
|
||||
opts['archive_mode'] = 'on'
|
||||
opts['archive_command'] = 'false'
|
||||
self.single_user_mode(options=opts)
|
||||
if self.rewind(leader):
|
||||
ret = self.start()
|
||||
else:
|
||||
logger.error("unable to rewind the former master")
|
||||
self.remove_data_directory()
|
||||
ret = True
|
||||
self._need_rewind = False
|
||||
change_role and ret and self.call_nowait(ACTION_ON_ROLE_CHANGE)
|
||||
return ret
|
||||
else:
|
||||
return True
|
||||
|
||||
change_role = self.role == 'master'
|
||||
self._need_rewind = (self._need_rewind or change_role) and self.can_rewind
|
||||
if self._need_rewind:
|
||||
logger.info("set the rewind flag after demote")
|
||||
self.write_recovery_conf(leader)
|
||||
if leader and self._need_rewind: # we have a leader and need to rewind
|
||||
if self.is_running():
|
||||
self.stop()
|
||||
# at present, pg_rewind only runs when the cluster is shut down cleanly
|
||||
# and not shutdown in recovery. We have to remove the recovery.conf if present
|
||||
# and start/shutdown in a single user mode to emulate this.
|
||||
# XXX: if recovery.conf is linked, it will be written anew as a normal file.
|
||||
if os.path.islink(self.recovery_conf):
|
||||
os.unlink(self.recovery_conf)
|
||||
else:
|
||||
os.remove(self.recovery_conf)
|
||||
# Archived segments might be useful to pg_rewind,
|
||||
# clean the flags that tell we should remove them.
|
||||
self.cleanup_archive_status()
|
||||
# Start in a single user mode and stop to produce a clean shutdown
|
||||
opts = self.read_postmaster_opts()
|
||||
opts.update({'archive_mode': 'on', 'archive_command': 'false'})
|
||||
self.single_user_mode(options=opts)
|
||||
if self.rewind(leader):
|
||||
ret = self.start()
|
||||
else:
|
||||
logger.error("unable to rewind the former master")
|
||||
self.remove_data_directory()
|
||||
ret = True
|
||||
self._need_rewind = False
|
||||
else: # do not rewind until the leader becomes available
|
||||
ret = self.restart()
|
||||
if change_role and ret:
|
||||
self.call_nowait(ACTION_ON_ROLE_CHANGE)
|
||||
return ret
|
||||
|
||||
def save_configuration_files(self):
|
||||
"""
|
||||
copy postgresql.conf to postgresql.conf.backup to be able to retrive configuration files
|
||||
@@ -609,18 +670,16 @@ recovery_target_timeline = 'latest'
|
||||
"""
|
||||
try:
|
||||
for f in self.configuration_to_save:
|
||||
if os.path.isfile(f):
|
||||
shutil.copy(f, f + '.backup')
|
||||
except IOError:
|
||||
os.path.isfile(f) and shutil.copy(f, f + '.backup')
|
||||
except:
|
||||
logger.exception('unable to create backup copies of configuration files')
|
||||
|
||||
def restore_configuration_files(self):
|
||||
""" restore a previously saved postgresql.conf """
|
||||
try:
|
||||
for f in self.configuration_to_save:
|
||||
if not os.path.isfile(f) and os.path.isfile(f + '.backup'):
|
||||
shutil.copy(f + '.backup', f)
|
||||
except IOError:
|
||||
not os.path.isfile(f) and os.path.isfile(f + '.backup') and shutil.copy(f + '.backup', f)
|
||||
except:
|
||||
logger.exception('unable to restore configuration files from backup')
|
||||
|
||||
def promote(self):
|
||||
@@ -634,6 +693,9 @@ recovery_target_timeline = 'latest'
|
||||
self.call_nowait(ACTION_ON_ROLE_CHANGE)
|
||||
return ret
|
||||
|
||||
def demote(self):
|
||||
self.follow_the_leader(None)
|
||||
|
||||
def create_or_update_role(self, name, password, options):
|
||||
self.query("""DO $$
|
||||
BEGIN
|
||||
@@ -647,10 +709,13 @@ BEGIN
|
||||
END;
|
||||
$$""".format(name, options), name, password, password)
|
||||
|
||||
def create_replication_user(self):
|
||||
self.create_or_update_role(self.replication['username'], self.replication['password'], 'REPLICATION')
|
||||
def create_replication_user(self, superuser=False):
|
||||
options = 'REPLICATION SUPERUSER' if superuser else 'REPLICATION'
|
||||
self.create_or_update_role(self.replication['username'], self.replication['password'], '{0}'.format(options))
|
||||
|
||||
def create_connection_user(self):
|
||||
def create_connection_users(self):
|
||||
if 'username' in self.superuser:
|
||||
self.create_or_update_role(self.superuser['username'], self.superuser['password'], 'SUPERUSER')
|
||||
if self.admin:
|
||||
self.create_or_update_role(self.admin['username'], self.admin['password'], 'CREATEDB CREATEROLE')
|
||||
|
||||
@@ -670,18 +735,7 @@ $$""".format(name, options), name, password, password)
|
||||
if self.use_slots:
|
||||
try:
|
||||
self.load_replication_slots()
|
||||
# if the replicatefrom tag is set on the member - we should not create the replication slot for it on
|
||||
# the current master, because that member would replicate from elsewhere. We still create the slot if
|
||||
# the replicatefrom destination member is currently not a member of the cluster (fallback to the
|
||||
# master), or if replicatefrom destination member happens to be the current master
|
||||
if self.role == 'master':
|
||||
slots = [m.name for m in cluster.members if m.name != self.name and
|
||||
(m.replicatefrom is None or m.replicatefrom == self.name or
|
||||
not cluster.has_member(m.replicatefrom))]
|
||||
else:
|
||||
# only manage slots for replicas that replicate from this one, except for the leader among them
|
||||
slots = [m.name for m in cluster.members if m.replicatefrom == self.name and
|
||||
m.name != cluster.leader.name]
|
||||
slots = [m.name for m in cluster.members if m.name != self.name] if self.role == 'master' else []
|
||||
# drop unused slots
|
||||
for slot in set(self.replication_slots) - set(slots):
|
||||
self.query("""SELECT pg_drop_replication_slot(%s)
|
||||
@@ -695,45 +749,34 @@ $$""".format(name, options), name, password, password)
|
||||
WHERE slot_name = %s)""", slot, slot)
|
||||
|
||||
self.replication_slots = slots
|
||||
except psycopg2.Error:
|
||||
except:
|
||||
logger.exception('Exception when changing replication slots')
|
||||
|
||||
def last_operation(self):
|
||||
return str(self.xlog_position())
|
||||
|
||||
def bootstrap(self, cluster_initialized=False, clone_member=None):
|
||||
def bootstrap(self, current_leader=None):
|
||||
"""
|
||||
Populate PostgreSQL data directory by doing one of the following:
|
||||
- create with initdb if there is no master.
|
||||
- initialize the replica from an existing member (master or replica)
|
||||
- initialize the replica using the replica creation method that
|
||||
works without the replication connection (i.e. restore from on-disk
|
||||
base backup)
|
||||
|
||||
The choice between the last 2 is triggered by the initialize flag.
|
||||
We should never try to initdb an already initialized cluster, nor
|
||||
try to bootstrap the cluster that lacks the initialize key using the
|
||||
master-less replica creation method (in the latter case, there is
|
||||
no clear inidicator of the moment we should abandon our attempts and
|
||||
swich to initdb).
|
||||
|
||||
Failure during initdb always leads to an exception, since there is
|
||||
no point in continuing if initdb fails. For the rest of the cases,
|
||||
the function returns False in order to inidicate a failed attempt
|
||||
that should be retried in the future.
|
||||
Initially bootstrap PostgreSQL, either by creating a data
|
||||
directory with initdb, or by initalizing a replica from an
|
||||
exiting leader. Failure in the first case always leads to
|
||||
exception, since there is no point in continuing if initdb failed.
|
||||
In the second case, however, a False is returned on failure, since
|
||||
it is normal for the replica to retry a failed attempt to initialize
|
||||
from the master.
|
||||
"""
|
||||
ret = False
|
||||
if not (cluster_initialized or clone_member):
|
||||
if not current_leader:
|
||||
ret = self.initialize() and self.start()
|
||||
if ret:
|
||||
self.create_replication_user()
|
||||
self.create_connection_user()
|
||||
self.create_connection_users()
|
||||
else:
|
||||
raise PostgresException("Could not bootstrap master PostgreSQL")
|
||||
else:
|
||||
if self.sync_replica(clone_member):
|
||||
if self.sync_from_leader(current_leader):
|
||||
self.restore_configuration_files()
|
||||
self.write_recovery_conf(clone_member, True)
|
||||
self.write_recovery_conf(current_leader)
|
||||
ret = self.start()
|
||||
return ret
|
||||
|
||||
@@ -743,7 +786,7 @@ $$""".format(name, options), name, password, password)
|
||||
new_name = '{0}_{1}'.format(self.data_dir, time.strftime('%Y-%m-%d-%H-%M-%S'))
|
||||
logger.info('renaming data directory to %s', new_name)
|
||||
os.rename(self.data_dir, new_name)
|
||||
except OSError:
|
||||
except:
|
||||
logger.exception("Could not rename data directory %s", self.data_dir)
|
||||
|
||||
def remove_data_directory(self):
|
||||
@@ -757,16 +800,16 @@ $$""".format(name, options), name, password, password)
|
||||
os.remove(self.data_dir)
|
||||
elif os.path.isdir(self.data_dir):
|
||||
shutil.rmtree(self.data_dir)
|
||||
except (IOError, OSError):
|
||||
except:
|
||||
logger.exception('Could not remove data directory %s', self.data_dir)
|
||||
self.move_data_directory()
|
||||
|
||||
def basebackup(self, clone_member, env):
|
||||
def basebackup(self, leader, env):
|
||||
# creates a replica data dir using pg_basebackup.
|
||||
# this is the default, built-in create_replica_method
|
||||
# tries twice, then returns failure (as 1)
|
||||
# uses "stream" as the xlog-method to avoid sync issues
|
||||
master_connection = clone_member.conn_url
|
||||
master_connection = leader.conn_url
|
||||
maxfailures = 2
|
||||
ret = 1
|
||||
for bbfailures in range(0, maxfailures):
|
||||
|
||||
@@ -9,7 +9,7 @@ import boto.ec2
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AWSConnection(object):
|
||||
class AWSConnection:
|
||||
def __init__(self, cluster_name):
|
||||
self.available = False
|
||||
self.cluster_name = cluster_name if cluster_name is not None else 'unknown'
|
||||
@@ -56,7 +56,7 @@ class AWSConnection(object):
|
||||
conn = boto.ec2.connect_to_region(self.region)
|
||||
conn.create_tags([self.instance_id], tags)
|
||||
except Exception as e:
|
||||
logger.info("could not set tags for EC2 instance %s: %s", self.instance_id, e)
|
||||
logger.info("could not set tags for EC2 instance {}: {}".format(self.instance_id, e))
|
||||
return False
|
||||
return True
|
||||
|
||||
@@ -66,7 +66,6 @@ class AWSConnection(object):
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
|
||||
if len(sys.argv) == 4 and sys.argv[1] in ('on_start', 'on_stop', 'on_role_change'):
|
||||
AWSConnection(cluster_name=sys.argv[3]).on_role_change(sys.argv[2])
|
||||
else:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#!/usr/bin/env python
|
||||
#!/usr/bin/python
|
||||
|
||||
# sample script to clone new replicas using WAL-E restore
|
||||
# falls back to pg_basebackup if WAL-E restore fails, or if
|
||||
@@ -36,12 +36,13 @@ import argparse
|
||||
if sys.hexversion >= 0x03000000:
|
||||
long = int
|
||||
|
||||
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WALERestore(object):
|
||||
|
||||
def __init__(self, scope, datadir, connstring, env_dir, threshold_mb, threshold_pct, use_iam, no_master):
|
||||
def __init__(self, scope, datadir, connstring, env_dir, threshold_mb, threshold_pct, use_iam):
|
||||
self.scope = scope
|
||||
self.master_connection = connstring
|
||||
self.data_dir = datadir
|
||||
@@ -50,7 +51,6 @@ class WALERestore(object):
|
||||
self.wal_e.threshold_mb = threshold_mb
|
||||
self.wal_e.threshold_pct = threshold_pct
|
||||
self.wal_e.iam_string = ' --aws-instance-profile ' if use_iam == 1 else ''
|
||||
self.no_master = no_master
|
||||
self.wal_e.cmd = 'envdir {0} wal-e {1} '.format(self.wal_e.dir, self.wal_e.iam_string)
|
||||
self.init_error = (not os.path.exists(self.wal_e.dir))
|
||||
|
||||
@@ -104,23 +104,24 @@ class WALERestore(object):
|
||||
lsn_offset = hex((long(backup_start_segment[16:32], 16) << 24) + long(backup_start_offset))[2:-1]
|
||||
|
||||
# construct the LSN from the segment and offset
|
||||
backup_start_lsn = '{0}/{1}'.format(lsn_segment, lsn_offset)
|
||||
backup_start_lsn = '{}/{}'.format(lsn_segment, lsn_offset)
|
||||
|
||||
conn = None
|
||||
cursor = None
|
||||
diff_in_bytes = long(backup_size)
|
||||
if not self.no_master:
|
||||
try:
|
||||
# get the difference in bytes between the current WAL location and the backup start offset
|
||||
with psycopg2.connect(self.master_connection) as con:
|
||||
con.autocommit = True
|
||||
with con.cursor() as cur:
|
||||
cur.execute("SELECT pg_xlog_location_diff(pg_current_xlog_location(), %s)", (backup_start_lsn,))
|
||||
diff_in_bytes = long(cur.fetchone()[0])
|
||||
except psycopg2.Error as e:
|
||||
logger.error('could not determine difference with the master location: %s', e)
|
||||
return False
|
||||
else:
|
||||
# always try to use WAL-E if base backup is available
|
||||
diff_in_bytes = 0
|
||||
try:
|
||||
# get the difference in bytes between the current WAL location and the backup start offset
|
||||
conn = psycopg2.connect(self.master_connection)
|
||||
conn.autocommit = True
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT pg_xlog_location_diff(pg_current_xlog_location(), %s)", (backup_start_lsn,))
|
||||
diff_in_bytes = long(cursor.fetchone()[0])
|
||||
except psycopg2.Error as e:
|
||||
logger.error('could not determine difference with the master location: {}'.format(e))
|
||||
return False
|
||||
finally:
|
||||
cursor and cursor.close()
|
||||
conn and conn.close()
|
||||
|
||||
# if the size of the accumulated WAL segments is more than a certan percentage of the backup size
|
||||
# or exceeds the pre-determined size - pg_basebackup is chosen instead.
|
||||
@@ -139,7 +140,6 @@ class WALERestore(object):
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
|
||||
parser = argparse.ArgumentParser(description='Script to image replicas using WAL-E')
|
||||
parser.add_argument('--scope', required=True)
|
||||
parser.add_argument('--role', required=False)
|
||||
@@ -150,15 +150,13 @@ def main():
|
||||
parser.add_argument('--threshold_megabytes', type=int, default=10240)
|
||||
parser.add_argument('--threshold_backup_size_percentage', type=int, default=30)
|
||||
parser.add_argument('--use_iam', type=int, default=0)
|
||||
parser.add_argument('--no_master', type=int, default=0)
|
||||
args = parser.parse_args()
|
||||
|
||||
# retry cloning in a loop
|
||||
for _ in range(0, args.retries + 1):
|
||||
for retry in range(0, args.retries + 1):
|
||||
restore = WALERestore(scope=args.scope, datadir=args.datadir, connstring=args.connstring,
|
||||
env_dir=args.envdir, threshold_mb=args.threshold_megabytes,
|
||||
threshold_pct=args.threshold_backup_size_percentage, use_iam=args.use_iam,
|
||||
no_master=args.no_master)
|
||||
threshold_pct=args.threshold_backup_size_percentage, use_iam=args.use_iam)
|
||||
ret = restore.run()
|
||||
if ret == 0:
|
||||
break
|
||||
|
||||
+40
-25
@@ -1,61 +1,76 @@
|
||||
import datetime
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
import pytz
|
||||
import dateutil.parser
|
||||
|
||||
from patroni.exceptions import PatroniException
|
||||
|
||||
__ignore_sigterm = False
|
||||
__interrupted_sleep = False
|
||||
__reap_children = False
|
||||
ignore_sigterm = False
|
||||
interrupted_sleep = False
|
||||
reap_children = False
|
||||
|
||||
_DATE_TIME_RE = re.compile(r'''^
|
||||
(?P<year>\d{4})\-(?P<month>\d{2})\-(?P<day>\d{2}) # date
|
||||
T
|
||||
(?P<hour>\d{2}):(?P<minute>\d{2}):(?P<second>\d{2})\.(?P<microsecond>\d{6}) # time
|
||||
\d*Z$''', re.X)
|
||||
|
||||
|
||||
def parse_datetime(time_str):
|
||||
"""
|
||||
>>> parse_datetime('2015-06-10T12:56:30.552539016Z')
|
||||
datetime.datetime(2015, 6, 10, 12, 56, 30, 552539)
|
||||
>>> parse_datetime('2015-06-10 12:56:30.552539016Z')
|
||||
"""
|
||||
m = _DATE_TIME_RE.match(time_str)
|
||||
if not m:
|
||||
return None
|
||||
p = dict((n, int(m.group(n))) for n in 'year month day hour minute second microsecond'.split(' '))
|
||||
return datetime.datetime(**p)
|
||||
|
||||
|
||||
def calculate_ttl(expiration):
|
||||
"""
|
||||
>>> calculate_ttl(None)
|
||||
>>> calculate_ttl('2015-06-10 12:56:30.552539016Z') < 0
|
||||
True
|
||||
>>> calculate_ttl('2015-06-10 12:56:30.552539016Z')
|
||||
>>> calculate_ttl('2015-06-10T12:56:30.552539016Z') < 0
|
||||
True
|
||||
>>> calculate_ttl('fail-06-10T12:56:30.552539016Z')
|
||||
"""
|
||||
if not expiration:
|
||||
return None
|
||||
try:
|
||||
expiration = dateutil.parser.parse(expiration)
|
||||
except (ValueError, TypeError):
|
||||
expiration = parse_datetime(expiration)
|
||||
if not expiration:
|
||||
return None
|
||||
now = datetime.datetime.now(pytz.utc)
|
||||
now = datetime.datetime.utcnow()
|
||||
return int((expiration - now).total_seconds())
|
||||
|
||||
|
||||
def sigterm_handler(signo, stack_frame):
|
||||
global __ignore_sigterm
|
||||
if not __ignore_sigterm:
|
||||
__ignore_sigterm = True
|
||||
global ignore_sigterm
|
||||
if not ignore_sigterm:
|
||||
ignore_sigterm = True
|
||||
sys.exit()
|
||||
|
||||
|
||||
def sigchld_handler(signo, stack_frame):
|
||||
global __interrupted_sleep, __reap_children
|
||||
__reap_children = __interrupted_sleep = True
|
||||
global interrupted_sleep, reap_children
|
||||
reap_children = interrupted_sleep = True
|
||||
|
||||
|
||||
def sleep(interval):
|
||||
global __interrupted_sleep
|
||||
global interrupted_sleep
|
||||
current_time = time.time()
|
||||
end_time = current_time + interval
|
||||
while current_time < end_time:
|
||||
__interrupted_sleep = False
|
||||
interrupted_sleep = False
|
||||
time.sleep(end_time - current_time)
|
||||
if not __interrupted_sleep: # we will ignore only sigchld
|
||||
if not interrupted_sleep: # we will ignore only sigchld
|
||||
break
|
||||
current_time = time.time()
|
||||
__interrupted_sleep = False
|
||||
interrupted_sleep = False
|
||||
|
||||
|
||||
def setup_signal_handlers():
|
||||
@@ -64,8 +79,8 @@ def setup_signal_handlers():
|
||||
|
||||
|
||||
def reap_children():
|
||||
global __reap_children
|
||||
if __reap_children:
|
||||
global reap_children
|
||||
if reap_children:
|
||||
try:
|
||||
while True:
|
||||
ret = os.waitpid(-1, os.WNOHANG)
|
||||
@@ -74,7 +89,7 @@ def reap_children():
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
__reap_children = False
|
||||
reap_children = False
|
||||
|
||||
|
||||
class RetryFailedError(PatroniException):
|
||||
@@ -82,7 +97,7 @@ class RetryFailedError(PatroniException):
|
||||
"""Raised when retrying an operation ultimately failed, after retrying the maximum number of attempts."""
|
||||
|
||||
|
||||
class Retry(object):
|
||||
class Retry:
|
||||
|
||||
"""Helper for retrying a method in the face of retry-able exceptions"""
|
||||
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
__version__ = '0.80'
|
||||
__version__ = '0.76'
|
||||
|
||||
@@ -17,7 +17,7 @@ class ZooKeeperError(DCSError):
|
||||
pass
|
||||
|
||||
|
||||
class ExhibitorEnsembleProvider(object):
|
||||
class ExhibitorEnsembleProvider:
|
||||
|
||||
TIMEOUT = 3.1
|
||||
|
||||
@@ -54,7 +54,7 @@ class ExhibitorEnsembleProvider(object):
|
||||
def _query_exhibitors(self, exhibitors):
|
||||
random.shuffle(exhibitors)
|
||||
for host in exhibitors:
|
||||
uri = 'http://{0}:{1}{2}'.format(host, self._exhibitor_port, self._uri_path)
|
||||
uri = 'http://{}:{}{}'.format(host, self._exhibitor_port, self._uri_path)
|
||||
try:
|
||||
response = requests.get(uri, timeout=self.TIMEOUT)
|
||||
return response.json()
|
||||
@@ -84,9 +84,9 @@ class ZooKeeper(AbstractDCS):
|
||||
hosts = self.exhibitor.zookeeper_hosts
|
||||
|
||||
self.client = KazooClient(hosts=hosts,
|
||||
timeout=(config.get('session_timeout') or 30),
|
||||
timeout=(config.get('session_timeout', None) or 30),
|
||||
command_retry={
|
||||
'deadline': (config.get('reconnect_timeout') or 10),
|
||||
'deadline': (config.get('reconnect_timeout', None) or 10),
|
||||
'max_delay': 1,
|
||||
'max_tries': -1},
|
||||
connection_retry={'max_delay': 1, 'max_tries': -1})
|
||||
@@ -190,8 +190,7 @@ class ZooKeeper(AbstractDCS):
|
||||
|
||||
def attempt_to_acquire_leader(self):
|
||||
ret = self._create(self.leader_path, self._name, makepath=True, ephemeral=True)
|
||||
if ret:
|
||||
logger.info('Could not take out TTL lock')
|
||||
ret or logger.info('Could not take out TTL lock')
|
||||
return ret
|
||||
|
||||
def set_failover_value(self, value, index=None):
|
||||
|
||||
+1
-1
@@ -2,4 +2,4 @@
|
||||
from patroni.ctl import ctl
|
||||
|
||||
if __name__ == '__main__':
|
||||
ctl(None)
|
||||
ctl()
|
||||
|
||||
+10
-2
@@ -4,7 +4,7 @@ scope: &scope batman
|
||||
restapi:
|
||||
listen: 127.0.0.1:8008
|
||||
connect_address: 127.0.0.1:8008
|
||||
# auth: 'username:password'
|
||||
#auth: 'username:password'
|
||||
# certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
|
||||
# keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
|
||||
etcd:
|
||||
@@ -26,6 +26,9 @@ etcd:
|
||||
# - host1
|
||||
# - host2
|
||||
# - host3
|
||||
bdr:
|
||||
enable: "on"
|
||||
database: "bdrtest"
|
||||
postgresql:
|
||||
name: postgresql0
|
||||
scope: *scope
|
||||
@@ -72,6 +75,7 @@ postgresql:
|
||||
- basebackup
|
||||
# - wal_e
|
||||
# commented-out example for wal-e provisioning
|
||||
#create_replica_method: wal_e, basebackup
|
||||
#wal_e:
|
||||
#command: /patroni/scripts/wale_restore.py
|
||||
#env_dir: /etc/wal-e.d/env
|
||||
@@ -85,15 +89,19 @@ postgresql:
|
||||
restore_command: cp ../wal_archive/%f %p
|
||||
parameters:
|
||||
archive_mode: "on"
|
||||
wal_level: hot_standby
|
||||
wal_level: logical
|
||||
archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
|
||||
max_wal_senders: 10
|
||||
wal_keep_segments: 8
|
||||
archive_timeout: 1800s
|
||||
max_replication_slots: 10
|
||||
max_worker_processes: 10
|
||||
track_commit_timestamp: 'on'
|
||||
shared_preload_libraries: 'bdr'
|
||||
hot_standby: "on"
|
||||
wal_log_hints: "on"
|
||||
tags:
|
||||
nofailover: False
|
||||
noloadbalance: False
|
||||
clonefrom: False
|
||||
replicatefrom: 127.0.0.1
|
||||
|
||||
+10
-3
@@ -4,7 +4,7 @@ scope: &scope batman
|
||||
restapi:
|
||||
listen: 127.0.0.1:8009
|
||||
connect_address: 127.0.0.1:8009
|
||||
# auth: 'username:password'
|
||||
#auth: 'username:password'
|
||||
# certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
|
||||
# keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
|
||||
etcd:
|
||||
@@ -26,6 +26,9 @@ etcd:
|
||||
# - host1
|
||||
# - host2
|
||||
# - host3
|
||||
bdr:
|
||||
enable: "on"
|
||||
database: "bdrtest"
|
||||
postgresql:
|
||||
name: postgresql1
|
||||
scope: *scope
|
||||
@@ -63,7 +66,7 @@ postgresql:
|
||||
password: rep-pass
|
||||
network: 127.0.0.1/32
|
||||
superuser:
|
||||
username: postgres
|
||||
user: postgres
|
||||
password: zalando
|
||||
admin:
|
||||
username: admin
|
||||
@@ -86,9 +89,12 @@ postgresql:
|
||||
restore_command: cp ../wal_archive/%f %p
|
||||
parameters:
|
||||
archive_mode: "on"
|
||||
wal_level: hot_standby
|
||||
wal_level: logical
|
||||
archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
|
||||
max_wal_senders: 10
|
||||
max_worker_processes: 10
|
||||
track_commit_timestamp: 'on'
|
||||
shared_preload_libraries: 'bdr'
|
||||
wal_keep_segments: 8
|
||||
archive_timeout: 1800s
|
||||
max_replication_slots: 10
|
||||
@@ -98,3 +104,4 @@ tags:
|
||||
nofailover: False
|
||||
noloadbalance: False
|
||||
clonefrom: False
|
||||
replicatefrom: 127.0.0.1
|
||||
|
||||
-101
@@ -1,101 +0,0 @@
|
||||
ttl: &ttl 30
|
||||
loop_wait: &loop_wait 10
|
||||
scope: &scope batman
|
||||
restapi:
|
||||
listen: 127.0.0.1:8010
|
||||
connect_address: 127.0.0.1:8010
|
||||
auth: 'username:password'
|
||||
# certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
|
||||
# keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
|
||||
etcd:
|
||||
scope: *scope
|
||||
ttl: *ttl
|
||||
host: 127.0.0.1:4001
|
||||
#discovery_srv: my-etcd.domain
|
||||
#zookeeper:
|
||||
# scope: *scope
|
||||
# session_timeout: *ttl
|
||||
# reconnect_timeout: *loop_wait
|
||||
# hosts:
|
||||
# - 127.0.0.1:2181
|
||||
# - 127.0.0.2:2181
|
||||
# exhibitor:
|
||||
# poll_interval: 300
|
||||
# port: 8181
|
||||
# hosts:
|
||||
# - host1
|
||||
# - host2
|
||||
# - host3
|
||||
postgresql:
|
||||
name: postgresql2
|
||||
scope: *scope
|
||||
listen: 127.0.0.1:5434
|
||||
connect_address: 127.0.0.1:5434
|
||||
data_dir: data/postgresql2
|
||||
maximum_lag_on_failover: 1048576 # 1 megabyte in bytes
|
||||
use_slots: True
|
||||
pgpass: /tmp/pgpass2
|
||||
initdb: ## We allow the following options to be passed on to initdb
|
||||
# - auth: authmethod
|
||||
# - auth-host: authmethod
|
||||
# - auth-local: authmethod
|
||||
- encoding: UTF8
|
||||
# - data-checksums # When pg_rewind is needed on 9.3, this needs to be enabled
|
||||
# - locale: locale
|
||||
# - lc-collate: locale
|
||||
# - lc-ctype: locale
|
||||
# - lc-messages: locale
|
||||
# - lc-monetary: locale
|
||||
# - lc-numeric: locale
|
||||
# - lc-time: locale
|
||||
# - text-search-config: CFG
|
||||
# - xlogdir: directory
|
||||
# - debug
|
||||
# - noclean
|
||||
pg_rewind:
|
||||
username: postgres
|
||||
password: zalando
|
||||
pg_hba:
|
||||
- host all all 0.0.0.0/0 md5
|
||||
- hostssl all all 0.0.0.0/0 md5
|
||||
replication:
|
||||
username: replicator
|
||||
password: rep-pass
|
||||
network: 127.0.0.1/32
|
||||
superuser:
|
||||
username: postgres
|
||||
password: zalando
|
||||
admin:
|
||||
username: admin
|
||||
password: admin
|
||||
# commented-out example for wal-e provisioning
|
||||
create_replica_method:
|
||||
- basebackup
|
||||
# - wal_e
|
||||
# commented-out example for wal-e provisioning
|
||||
#wal_e:
|
||||
#command: /patroni/scripts/wale_restore.py
|
||||
#env_dir: /home/postgres/etc/wal-e.d/env
|
||||
#threshold_megabytes: 10240
|
||||
#threshold_backup_size_percentage: 30
|
||||
#retries: 2
|
||||
#use_iam: 1
|
||||
#recovery_conf:
|
||||
#restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1
|
||||
recovery_conf:
|
||||
restore_command: cp ../wal_archive/%f %p
|
||||
parameters:
|
||||
archive_mode: "on"
|
||||
wal_level: hot_standby
|
||||
archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
|
||||
max_wal_senders: 10
|
||||
wal_keep_segments: 8
|
||||
archive_timeout: 1800s
|
||||
max_replication_slots: 10
|
||||
hot_standby: "on"
|
||||
wal_log_hints: "on"
|
||||
tags:
|
||||
nofailover: False
|
||||
noloadbalance: False
|
||||
clonefrom: False
|
||||
replicatefrom: postgresql1
|
||||
@@ -1,11 +1,11 @@
|
||||
boto
|
||||
dnspython
|
||||
mock
|
||||
psycopg2>=2.6.1
|
||||
PyYAML
|
||||
requests
|
||||
six >= 1.7
|
||||
kazoo>=2.2.1
|
||||
python-etcd==0.4.3
|
||||
python-etcd>=0.4.1
|
||||
click>=4.1
|
||||
prettytable>=0.7
|
||||
tzlocal
|
||||
python-dateutil
|
||||
@@ -0,0 +1,11 @@
|
||||
boto
|
||||
mock
|
||||
dnspython3
|
||||
psycopg2>=2.6.1
|
||||
PyYAML
|
||||
requests
|
||||
six
|
||||
kazoo>=2.2.1
|
||||
python-etcd>=0.4.1
|
||||
click>=4.1
|
||||
prettytable>=0.7
|
||||
@@ -51,8 +51,8 @@ CLASSIFIERS = [
|
||||
'Operating System :: POSIX :: Linux',
|
||||
'Programming Language :: Python',
|
||||
'Programming Language :: Python :: 2.7',
|
||||
'Programming Language :: Python :: 3.3',
|
||||
'Programming Language :: Python :: 3.4',
|
||||
'Programming Language :: Python :: 3.5',
|
||||
'Programming Language :: Python :: Implementation :: CPython',
|
||||
]
|
||||
|
||||
@@ -76,7 +76,7 @@ class PyTest(TestCommand):
|
||||
def finalize_options(self):
|
||||
TestCommand.finalize_options(self)
|
||||
if self.cov_xml or self.cov_html:
|
||||
self.cov = ['--cov', MAIN_PACKAGE, '--cov-report', 'term-missing']
|
||||
self.cov = ['--cov', MAIN_PACKAGE, '--cov', MAIN_PACKAGE, '--cov-report', 'term-missing']
|
||||
if self.cov_xml:
|
||||
self.cov.extend(['--cov-report', 'xml'])
|
||||
if self.cov_html:
|
||||
@@ -116,7 +116,8 @@ def setup_package():
|
||||
# Some helper variables
|
||||
version = os.getenv('GO_PIPELINE_LABEL', VERSION)
|
||||
|
||||
install_reqs = get_install_requirements('requirements.txt')
|
||||
requirements = 'requirements-py2.txt' if sys.version_info[0] == 2 else 'requirements-py3.txt'
|
||||
install_reqs = get_install_requirements(requirements)
|
||||
|
||||
command_options = {'test': {'test_suite': ('setup.py', 'tests')}}
|
||||
if JUNIT_XML:
|
||||
@@ -141,9 +142,9 @@ def setup_package():
|
||||
packages=setuptools.find_packages(exclude=['tests', 'tests.*']),
|
||||
package_data={MAIN_PACKAGE: ["*.json"]},
|
||||
install_requires=install_reqs,
|
||||
setup_requires=['flake8'],
|
||||
setup_requires=['six', 'flake8'],
|
||||
cmdclass=cmdclass,
|
||||
tests_require=['mock', 'pytest-cov', 'pytest'],
|
||||
tests_require=['pytest-cov', 'pytest'],
|
||||
command_options=command_options,
|
||||
entry_points={'console_scripts': CONSOLE_SCRIPTS},
|
||||
)
|
||||
|
||||
+17
-51
@@ -16,15 +16,11 @@ class MockPostgresql(Mock):
|
||||
name = 'test'
|
||||
state = 'running'
|
||||
role = 'master'
|
||||
server_version = '999999'
|
||||
scope = 'dummy'
|
||||
|
||||
@staticmethod
|
||||
def connection():
|
||||
def connection(self):
|
||||
return psycopg2_connect()
|
||||
|
||||
@staticmethod
|
||||
def is_running():
|
||||
def is_running(self):
|
||||
return True
|
||||
|
||||
|
||||
@@ -33,37 +29,31 @@ class MockHa(Mock):
|
||||
dcs = Mock()
|
||||
state_handler = MockPostgresql()
|
||||
|
||||
@staticmethod
|
||||
def schedule_restart():
|
||||
def schedule_restart(self):
|
||||
return 'restart'
|
||||
|
||||
@staticmethod
|
||||
def schedule_reinitialize():
|
||||
def schedule_reinitialize(self):
|
||||
return 'reinitialize'
|
||||
|
||||
@staticmethod
|
||||
def restart():
|
||||
def restart(self):
|
||||
return (True, '')
|
||||
|
||||
@staticmethod
|
||||
def restart_scheduled():
|
||||
def restart_scheduled(self):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def fetch_nodes_statuses(members):
|
||||
def fetch_nodes_statuses(self, members):
|
||||
return [[None, True, None, None, {}]]
|
||||
|
||||
|
||||
class MockPatroni(Mock):
|
||||
class MockPatroni:
|
||||
|
||||
postgresql = MockPostgresql()
|
||||
ha = MockHa()
|
||||
dcs = Mock()
|
||||
tags = {}
|
||||
version = '0.00'
|
||||
|
||||
|
||||
class MockRequest(object):
|
||||
class MockRequest:
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
@@ -101,10 +91,10 @@ class TestRestApiHandler(unittest.TestCase):
|
||||
MockRestApiServer(RestApiHandler, b'GET /master')
|
||||
with patch.object(MockHa, 'restart_scheduled', Mock(return_value=True)):
|
||||
MockRestApiServer(RestApiHandler, b'GET /master')
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /master'))
|
||||
MockRestApiServer(RestApiHandler, b'GET /master')
|
||||
|
||||
def test_do_OPTIONS(self):
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0'))
|
||||
MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0')
|
||||
|
||||
with patch.object(BaseHTTPRequestHandler, 'handle_one_request') as mock_handle_request:
|
||||
mock_handle_request.side_effect = socket.error("foo")
|
||||
@@ -118,15 +108,15 @@ class TestRestApiHandler(unittest.TestCase):
|
||||
MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0')
|
||||
|
||||
def test_do_GET_patroni(self):
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /patroni'))
|
||||
MockRestApiServer(RestApiHandler, b'GET /patroni')
|
||||
|
||||
def test_basicauth(self):
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0'))
|
||||
MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0')
|
||||
MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0\nAuthorization:')
|
||||
|
||||
def test_do_POST_restart(self):
|
||||
request = b'POST /restart HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0'
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
with patch.object(MockHa, 'restart', Mock(side_effect=Exception)):
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
@@ -140,23 +130,19 @@ class TestRestApiHandler(unittest.TestCase):
|
||||
with patch.object(MockHa, 'schedule_reinitialize', Mock(return_value=None)):
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
cluster.leader.name = 'test'
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
@patch('time.sleep', Mock())
|
||||
def test_RestApiServer_query(self):
|
||||
with patch.object(MockCursor, 'execute', Mock(side_effect=psycopg2.OperationalError)):
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /patroni'))
|
||||
MockRestApiServer(RestApiHandler, b'GET /patroni')
|
||||
with patch.object(MockPostgresql, 'connection', Mock(side_effect=psycopg2.OperationalError)):
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /patroni'))
|
||||
MockRestApiServer(RestApiHandler, b'GET /patroni')
|
||||
|
||||
@patch('time.sleep', Mock())
|
||||
@patch.object(MockHa, 'dcs')
|
||||
def test_do_POST_failover(self, dcs):
|
||||
cluster = dcs.get_cluster.return_value
|
||||
|
||||
request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
|
||||
b'Content-Length: 0\n\n'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
|
||||
b'Content-Length: 25\n\n{"leader": "postgresql1"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
@@ -180,23 +166,3 @@ class TestRestApiHandler(unittest.TestCase):
|
||||
request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
|
||||
b'Content-Length: 50\n\n{"leader": "postgresql1", "member": "postgresql2"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
# Valid future date
|
||||
request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\
|
||||
b'"postgresql1", "member": "postgresql2", "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
# Exception: No timezone specified
|
||||
request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 97\n\n{"leader": ' +\
|
||||
b'"postgresql1", "member": "postgresql2", "scheduled_at": "6016-02-15T18:13:30.568224"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
# Exception: Scheduled in the past
|
||||
request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\
|
||||
b'"postgresql1", "member": "postgresql2", "scheduled_at": "1016-02-15T18:13:30.568224+01:00"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
# Invalid date
|
||||
request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\
|
||||
b'"postgresql1", "member": "postgresql2", "scheduled_at": "2010-02-29T18:13:30.568224+01:00"}'
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
|
||||
|
||||
+18
-18
@@ -1,15 +1,12 @@
|
||||
import boto.ec2
|
||||
import requests
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
from mock import Mock, patch
|
||||
import requests
|
||||
import boto.ec2
|
||||
from collections import namedtuple
|
||||
from patroni.scripts.aws import AWSConnection, main as _main
|
||||
from patroni.scripts.aws import AWSConnection
|
||||
from requests.exceptions import RequestException
|
||||
|
||||
|
||||
class MockEc2Connection(object):
|
||||
class MockEc2Connection:
|
||||
|
||||
def __init__(self, error=False):
|
||||
self.error = error
|
||||
@@ -26,7 +23,7 @@ class MockEc2Connection(object):
|
||||
return True
|
||||
|
||||
|
||||
class MockResponse(object):
|
||||
class MockResponse:
|
||||
|
||||
def __init__(self, content):
|
||||
self.content = content
|
||||
@@ -38,6 +35,15 @@ class MockResponse(object):
|
||||
|
||||
class TestAWSConnection(unittest.TestCase):
|
||||
|
||||
def __init__(self, method_name='runTest'):
|
||||
super(TestAWSConnection, self).__init__(method_name)
|
||||
|
||||
def set_error(self):
|
||||
self.error = True
|
||||
|
||||
def set_json_error(self):
|
||||
self.json_error = True
|
||||
|
||||
def boto_ec2_connect_to_region(self, region):
|
||||
return MockEc2Connection(self.error)
|
||||
|
||||
@@ -68,27 +74,21 @@ class TestAWSConnection(unittest.TestCase):
|
||||
self.assertTrue(self.conn.on_role_change('master'))
|
||||
|
||||
def test_non_aws(self):
|
||||
self.error = True
|
||||
self.set_error()
|
||||
conn = AWSConnection('test')
|
||||
self.assertFalse(conn.aws_available())
|
||||
self.assertFalse(conn._tag_ebs('master'))
|
||||
self.assertFalse(conn._tag_ec2('master'))
|
||||
|
||||
def test_aws_bizare_response(self):
|
||||
self.json_error = True
|
||||
self.set_json_error()
|
||||
conn = AWSConnection('test')
|
||||
self.assertFalse(conn.aws_available())
|
||||
|
||||
def test_aws_tag_ebs_error(self):
|
||||
self.error = True
|
||||
self.set_error()
|
||||
self.assertFalse(self.conn._tag_ebs("master"))
|
||||
|
||||
def test_aws_tag_ec2_error(self):
|
||||
self.error = True
|
||||
self.set_error()
|
||||
self.assertFalse(self.conn._tag_ec2("master"))
|
||||
|
||||
@patch('sys.exit', Mock())
|
||||
def test_main(self):
|
||||
self.assertIsNone(_main())
|
||||
sys.argv = ['aws.py', 'on_start', 'replica', 'foo']
|
||||
self.assertIsNone(_main())
|
||||
|
||||
+153
-167
@@ -1,28 +1,31 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
import pytest
|
||||
import requests.exceptions
|
||||
import unittest
|
||||
import psycopg2
|
||||
import requests
|
||||
import patroni.exceptions
|
||||
import etcd
|
||||
from mock import patch, Mock
|
||||
|
||||
from click.testing import CliRunner
|
||||
from etcd import EtcdException
|
||||
from mock import patch, Mock, MagicMock
|
||||
from patroni.ctl import ctl, members, store_config, load_config, output_members, post_patroni, get_dcs, \
|
||||
wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure
|
||||
from patroni.ha import Ha
|
||||
from patroni.etcd import Etcd, Client
|
||||
from patroni.exceptions import PatroniCtlException
|
||||
from psycopg2 import OperationalError
|
||||
from test_etcd import etcd_read, etcd_write, requests_get, socket_getaddrinfo, MockResponse
|
||||
from test_zookeeper import MockKazooClient
|
||||
from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \
|
||||
get_cluster_initialized_with_only_leader
|
||||
get_cluster_initialized_with_only_leader, MockPostgresql, MockPatroni, run_async, \
|
||||
get_cluster_not_initialized_without_leader
|
||||
from test_etcd import etcd_read, etcd_write, requests_get, MockResponse
|
||||
from test_postgresql import MockConnect, psycopg2_connect
|
||||
|
||||
CONFIG_FILE_PATH = './test-ctl.yaml'
|
||||
|
||||
|
||||
def test_rw_config():
|
||||
runner = CliRunner()
|
||||
config = {'a': 'b'}
|
||||
config = {'a':'b'}
|
||||
with runner.isolated_filesystem():
|
||||
store_config(config, CONFIG_FILE_PATH + '/dummy')
|
||||
os.remove(CONFIG_FILE_PATH + '/dummy')
|
||||
@@ -42,36 +45,43 @@ def test_rw_config():
|
||||
load_config(CONFIG_FILE_PATH, None)
|
||||
load_config(CONFIG_FILE_PATH, '0.0.0.0')
|
||||
|
||||
|
||||
@patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'etcd', 'hostname': 'localhost', 'port': 4001}}))
|
||||
class TestCtl(unittest.TestCase):
|
||||
|
||||
@patch('socket.getaddrinfo', socket_getaddrinfo)
|
||||
def setUp(self):
|
||||
self.runner = CliRunner()
|
||||
with patch.object(Client, 'machines') as mock_machines:
|
||||
mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
|
||||
self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
|
||||
self.e.client.read = etcd_read
|
||||
self.e.client.write = etcd_write
|
||||
self.e.client.delete = Mock(side_effect=EtcdException)
|
||||
@patch.object(Client, 'machines')
|
||||
def setUp(self, mock_machines):
|
||||
mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
|
||||
self.p = MockPostgresql()
|
||||
self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
|
||||
self.e.client.read = etcd_read
|
||||
self.e.client.write = etcd_write
|
||||
self.e.client.delete = Mock(side_effect=etcd.EtcdException())
|
||||
self.ha = Ha(MockPatroni(self.p, self.e))
|
||||
self.ha._async_executor.run_async = run_async
|
||||
self.ha.old_cluster = self.e.get_cluster()
|
||||
self.ha.cluster = get_cluster_not_initialized_without_leader()
|
||||
self.ha.load_cluster_from_dcs = Mock()
|
||||
|
||||
@patch('psycopg2.connect', psycopg2_connect)
|
||||
def test_get_cursor(self):
|
||||
self.assertIsNone(get_cursor(get_cluster_initialized_without_leader(), role='master'))
|
||||
c = get_cursor(get_cluster_initialized_without_leader(), role='master')
|
||||
assert c is None
|
||||
|
||||
self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), role='master'))
|
||||
c = get_cursor(get_cluster_initialized_with_leader(), role='master')
|
||||
assert c is not None
|
||||
|
||||
# MockCursor returns pg_is_in_recovery as false
|
||||
self.assertIsNone(get_cursor(get_cluster_initialized_with_leader(), role='replica'))
|
||||
c = get_cursor(get_cluster_initialized_with_leader(), role='replica')
|
||||
# # MockCursor returns pg_is_in_recovery as false
|
||||
assert c is None
|
||||
|
||||
self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), role='any'))
|
||||
c = get_cursor(get_cluster_initialized_with_leader(), role='any')
|
||||
assert c is not None
|
||||
|
||||
def test_output_members(self):
|
||||
cluster = get_cluster_initialized_with_leader()
|
||||
self.assertIsNone(output_members(cluster, name='abc', fmt='pretty'))
|
||||
self.assertIsNone(output_members(cluster, name='abc', fmt='json'))
|
||||
self.assertIsNone(output_members(cluster, name='abc', fmt='tsv'))
|
||||
output_members(cluster, name='abc', format='pretty')
|
||||
output_members(cluster, name='abc', format='json')
|
||||
output_members(cluster, name='abc', format='tsv')
|
||||
|
||||
@patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
|
||||
@patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
|
||||
@@ -81,112 +91,89 @@ class TestCtl(unittest.TestCase):
|
||||
@patch('requests.post', requests_get)
|
||||
@patch('patroni.ctl.post_patroni', Mock(return_value=MockResponse()))
|
||||
def test_failover(self):
|
||||
runner = CliRunner()
|
||||
|
||||
with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())):
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
other
|
||||
|
||||
y''')
|
||||
assert 'leader' in result.output
|
||||
assert 'Failing over to new leader' in result.output
|
||||
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
other
|
||||
2100-01-01T12:23:00
|
||||
y''')
|
||||
assert result.exit_code == 0
|
||||
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
other
|
||||
2030-01-01T12:23:00
|
||||
y''')
|
||||
assert result.exit_code == 0
|
||||
|
||||
# Aborting failover,as we anser NO to the confirmation
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
other
|
||||
|
||||
N''')
|
||||
assert result.exit_code == 1
|
||||
assert 'Aborting failover' in str(result.exception)
|
||||
|
||||
# Target and source are equal
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
leader
|
||||
|
||||
y''')
|
||||
assert result.exit_code == 1
|
||||
assert 'target and source are the same' in str(result.exception)
|
||||
|
||||
# Reality is not part of this cluster
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
Reality
|
||||
|
||||
y''')
|
||||
assert result.exit_code == 1
|
||||
assert 'Reality does not exist' in str(result.exception)
|
||||
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--force'])
|
||||
assert 'Member' in result.output
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--force'])
|
||||
assert 'Failing over to new leader' in result.output
|
||||
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--force',
|
||||
'--scheduled', '2015-01-01T12:00:00+01:00'])
|
||||
assert result.exit_code == 0
|
||||
|
||||
# Invalid timestamp
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', 'invalid'])
|
||||
assert result.exit_code != 0
|
||||
|
||||
# Invalid timestamp
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--force',
|
||||
'--scheduled', '2115-02-30T12:00:00+01:00'])
|
||||
assert result.exit_code != 0
|
||||
|
||||
# Specifying wrong leader
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='dummy')
|
||||
assert result.exit_code == 1
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='dummy')
|
||||
assert 'is not the leader of cluster' in str(result.exception)
|
||||
|
||||
with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_only_leader())):
|
||||
# No members available
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
other
|
||||
|
||||
y''')
|
||||
assert result.exit_code == 1
|
||||
assert 'No candidates found to failover to' in str(result.exception)
|
||||
|
||||
with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_without_leader())):
|
||||
# No master available
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
other
|
||||
|
||||
y''')
|
||||
assert result.exit_code == 1
|
||||
assert 'This cluster has no master' in str(result.exception)
|
||||
|
||||
with patch('patroni.ctl.post_patroni', Mock(side_effect=Exception())):
|
||||
# Non-responding patroni
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
other
|
||||
|
||||
y''')
|
||||
assert 'falling back to DCS' in result.output
|
||||
assert 'Failover failed' in result.output
|
||||
|
||||
mocked = Mock()
|
||||
mocked.return_value.status_code = 500
|
||||
with patch('patroni.ctl.post_patroni', Mock(return_value=mocked)):
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
other
|
||||
|
||||
y''')
|
||||
assert 'Failover failed' in result.output
|
||||
assert 'Failover failed, details' in result.output
|
||||
|
||||
@patch('patroni.zookeeper.KazooClient', MockKazooClient)
|
||||
@patch('requests.get', requests_get)
|
||||
def test_get_dcs(self):
|
||||
self.assertIsNotNone(get_dcs({'dcs': {'scheme': 'zookeeper', 'hostname': 'foo', 'port': 2181}}, 'dummy'))
|
||||
self.assertIsNotNone(get_dcs({'dcs': {'scheme': 'exhibitor', 'hostname': 'exhibitor', 'port': 8181}}, 'dummy'))
|
||||
self.assertRaises(PatroniCtlException, get_dcs, {'scheme': 'dummy'}, 'dummy')
|
||||
# with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())):
|
||||
# result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='nonsense')
|
||||
# assert 'is not the leader of cluster' in str(result.exception)
|
||||
|
||||
# result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8', '--master', 'nonsense'])
|
||||
# assert 'is not the leader of cluster' in str(result.exception)
|
||||
|
||||
# result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='leader\nother\nn')
|
||||
# assert 'Aborting failover' in str(result.exception)
|
||||
|
||||
# with patch('patroni.ctl.wait_for_leader', Mock(return_value = get_cluster_initialized_with_leader())):
|
||||
# result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='leader\nother\nY')
|
||||
# assert 'master did not change after' in result.output
|
||||
|
||||
# result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='leader\nother\nY')
|
||||
# assert 'Failover failed' in result.output
|
||||
|
||||
def test_(self):
|
||||
self.assertRaises(patroni.exceptions.PatroniCtlException, get_dcs, {'scheme': 'dummy'}, 'dummy')
|
||||
|
||||
@patch('psycopg2.connect', psycopg2_connect)
|
||||
@patch('patroni.ctl.query_member', Mock(return_value=([['mock column']], None)))
|
||||
def test_query(self):
|
||||
runner = CliRunner()
|
||||
|
||||
with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)):
|
||||
# Mutually exclusive
|
||||
result = self.runner.invoke(ctl, [
|
||||
result = runner.invoke(ctl, [
|
||||
'query',
|
||||
'alpha',
|
||||
'--member',
|
||||
@@ -194,14 +181,14 @@ y''')
|
||||
'--role',
|
||||
'master',
|
||||
])
|
||||
assert result.exit_code == 1
|
||||
assert 'mutually exclusive' in str(result.exception)
|
||||
|
||||
with self.runner.isolated_filesystem():
|
||||
with open('dummy', 'w') as dummy_file:
|
||||
dummy_file.write('SELECT 1')
|
||||
with runner.isolated_filesystem():
|
||||
dummy_file = open('dummy', 'w')
|
||||
dummy_file.write('SELECT 1')
|
||||
dummy_file.close()
|
||||
|
||||
# Mutually exclusive
|
||||
result = self.runner.invoke(ctl, [
|
||||
result = runner.invoke(ctl, [
|
||||
'query',
|
||||
'alpha',
|
||||
'--file',
|
||||
@@ -209,52 +196,45 @@ y''')
|
||||
'--command',
|
||||
'dummy',
|
||||
])
|
||||
assert result.exit_code == 1
|
||||
assert 'mutually exclusive' in str(result.exception)
|
||||
|
||||
result = self.runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy'])
|
||||
result = runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy'])
|
||||
|
||||
os.remove('dummy')
|
||||
|
||||
result = self.runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1'])
|
||||
assert 'mock column' in result.output
|
||||
|
||||
# --command or --file is mandatory
|
||||
result = self.runner.invoke(ctl, ['query', 'alpha'])
|
||||
assert result.exit_code == 1
|
||||
|
||||
result = self.runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1', '--username', 'root',
|
||||
'--password', '--dbname', 'postgres'], input='ab\nab')
|
||||
result = runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1'])
|
||||
assert 'mock column' in result.output
|
||||
|
||||
@patch('patroni.ctl.get_cursor', Mock(return_value=MockConnect().cursor()))
|
||||
def test_query_member(self):
|
||||
rows = query_member(None, None, None, 'master', 'SELECT pg_is_in_recovery()')
|
||||
self.assertTrue('False' in str(rows))
|
||||
assert 'False' in str(rows)
|
||||
|
||||
rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
|
||||
self.assertEquals(rows, (None, None))
|
||||
assert rows == (None, None)
|
||||
|
||||
with patch('patroni.ctl.get_cursor', Mock(return_value=None)):
|
||||
rows = query_member(None, None, None, None, 'SELECT pg_is_in_recovery()')
|
||||
self.assertTrue('No connection to' in str(rows))
|
||||
assert 'No connection to' in str(rows)
|
||||
|
||||
rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
|
||||
self.assertTrue('No connection to' in str(rows))
|
||||
assert 'No connection to' in str(rows)
|
||||
|
||||
with patch('patroni.ctl.get_cursor', Mock(side_effect=OperationalError('bla'))):
|
||||
with patch('patroni.ctl.get_cursor', Mock(side_effect=psycopg2.OperationalError('bla'))):
|
||||
rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
|
||||
|
||||
with patch('test_postgresql.MockCursor.execute', Mock(side_effect=OperationalError('bla'))):
|
||||
with patch('test_postgresql.MockCursor.execute', Mock(side_effect=psycopg2.OperationalError('bla'))):
|
||||
rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
|
||||
|
||||
@patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
|
||||
def test_dsn(self):
|
||||
runner = CliRunner()
|
||||
|
||||
with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)):
|
||||
result = self.runner.invoke(ctl, ['dsn', 'alpha', '--dcs', '8.8.8.8'])
|
||||
result = runner.invoke(ctl, ['dsn', 'alpha', '--dcs', '8.8.8.8'])
|
||||
assert 'host=127.0.0.1 port=5435' in result.output
|
||||
|
||||
# Mutually exclusive options
|
||||
result = self.runner.invoke(ctl, [
|
||||
result = runner.invoke(ctl, [
|
||||
'dsn',
|
||||
'alpha',
|
||||
'--role',
|
||||
@@ -262,29 +242,26 @@ y''')
|
||||
'--member',
|
||||
'dummy',
|
||||
])
|
||||
assert result.exit_code == 1
|
||||
assert 'mutually exclusive' in str(result.exception)
|
||||
|
||||
# Non-existing member
|
||||
result = self.runner.invoke(ctl, ['dsn', 'alpha', '--member', 'dummy'])
|
||||
assert result.exit_code == 1
|
||||
result = runner.invoke(ctl, ['dsn', 'alpha', '--member', 'dummy'])
|
||||
assert 'Can not find' in str(result.exception)
|
||||
|
||||
# result = runner.invoke(ctl, ['dsn', 'alpha', '--dcs', '8.8.8.8', '--role', 'replica'])
|
||||
# assert 'host=127.0.0.1 port=5436' in result.output
|
||||
|
||||
@patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
|
||||
@patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
|
||||
@patch('requests.get', requests_get)
|
||||
@patch('requests.post', requests_get)
|
||||
def test_restart_reinit(self):
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y')
|
||||
assert result.exit_code == 0
|
||||
runner = CliRunner()
|
||||
|
||||
result = self.runner.invoke(ctl, ['reinit', 'alpha', '--dcs', '8.8.8.8'], input='y')
|
||||
assert result.exit_code == 1
|
||||
result = runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y')
|
||||
result = runner.invoke(ctl, ['reinit', 'alpha', '--dcs', '8.8.8.8'], input='y')
|
||||
|
||||
# Aborted restart
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='N')
|
||||
assert result.exit_code == 1
|
||||
|
||||
# Not a member
|
||||
result = self.runner.invoke(ctl, [
|
||||
result = runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='N')
|
||||
result = runner.invoke(ctl, [
|
||||
'restart',
|
||||
'alpha',
|
||||
'--dcs',
|
||||
@@ -292,93 +269,100 @@ y''')
|
||||
'dummy',
|
||||
'--any',
|
||||
], input='y')
|
||||
assert result.exit_code == 1
|
||||
assert 'not a member' in str(result.exception)
|
||||
|
||||
with patch('requests.post', Mock(return_value=MockResponse())):
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y')
|
||||
result = runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y')
|
||||
|
||||
@patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
|
||||
@patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
|
||||
def test_remove(self):
|
||||
result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='alpha\nslave')
|
||||
runner = CliRunner()
|
||||
|
||||
result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='alpha\nslave')
|
||||
assert 'Please confirm' in result.output
|
||||
assert 'You are about to remove all' in result.output
|
||||
# Not typing an exact confirmation
|
||||
assert result.exit_code == 1
|
||||
assert 'You did not exactly type' in str(result.exception)
|
||||
|
||||
# master specified does not match master of cluster
|
||||
result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='''alpha
|
||||
result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='''alpha
|
||||
Yes I am aware
|
||||
slave''')
|
||||
assert result.exit_code == 1
|
||||
assert 'You did not specify the current master of the cluster' in str(result.exception)
|
||||
|
||||
# cluster specified on cmdline does not match verification prompt
|
||||
result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='beta\nleader')
|
||||
assert result.exit_code == 1
|
||||
result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='beta\nleader')
|
||||
assert 'Cluster names specified do not match' in str(result.exception)
|
||||
|
||||
with patch('patroni.etcd.Etcd.get_cluster', get_cluster_initialized_with_leader):
|
||||
result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'],
|
||||
input='''alpha
|
||||
result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'],
|
||||
input='''alpha
|
||||
Yes I am aware
|
||||
leader''')
|
||||
assert 'object has no attribute' in str(result.exception)
|
||||
|
||||
with patch('patroni.ctl.get_dcs', Mock(return_value=Mock())):
|
||||
# Not implemented DCS
|
||||
result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='''alpha
|
||||
result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'],
|
||||
input='''alpha
|
||||
Yes I am aware
|
||||
leader''')
|
||||
assert result.exit_code == 1
|
||||
assert 'We have not implemented this for DCS of type' in str(result.exception)
|
||||
|
||||
@patch('patroni.etcd.Etcd.watch', Mock(return_value=None))
|
||||
@patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
|
||||
def test_wait_for_leader(self):
|
||||
dcs = self.e
|
||||
self.assertRaises(PatroniCtlException, wait_for_leader, dcs, 0)
|
||||
self.assertRaises(patroni.exceptions.PatroniCtlException, wait_for_leader, dcs, 0)
|
||||
|
||||
cluster = wait_for_leader(dcs=dcs, timeout=2)
|
||||
assert cluster.leader.member.name == 'leader'
|
||||
|
||||
def test_post_patroni(self):
|
||||
with patch('requests.post', MagicMock(side_effect=requests.exceptions.ConnectionError('foo'))):
|
||||
member = get_cluster_initialized_with_leader().leader.member
|
||||
self.assertRaises(requests.exceptions.ConnectionError, post_patroni, member, 'dummy', {})
|
||||
member = get_cluster_initialized_with_leader().leader.member
|
||||
self.assertRaises(requests.exceptions.ConnectionError, post_patroni, member, 'dummy', {})
|
||||
|
||||
def test_ctl(self):
|
||||
self.runner.invoke(ctl, ['list'])
|
||||
runner = CliRunner()
|
||||
|
||||
result = self.runner.invoke(ctl, ['--help'])
|
||||
runner.invoke(ctl, ['list'])
|
||||
|
||||
result = runner.invoke(ctl, ['--help'])
|
||||
assert 'Usage:' in result.output
|
||||
|
||||
def test_get_any_member(self):
|
||||
self.assertIsNone(get_any_member(get_cluster_initialized_without_leader(), role='master'))
|
||||
m = get_any_member(get_cluster_initialized_without_leader(), role='master')
|
||||
assert m is None
|
||||
|
||||
m = get_any_member(get_cluster_initialized_with_leader(), role='master')
|
||||
self.assertEquals(m.name, 'leader')
|
||||
assert m.name == 'leader'
|
||||
|
||||
def test_get_all_members(self):
|
||||
self.assertEquals(list(get_all_members(get_cluster_initialized_without_leader(), role='master')), [])
|
||||
r = list(get_all_members(get_cluster_initialized_without_leader(), role='master'))
|
||||
assert len(r) == 0
|
||||
|
||||
r = list(get_all_members(get_cluster_initialized_with_leader(), role='master'))
|
||||
self.assertEquals(len(r), 1)
|
||||
self.assertEquals(r[0].name, 'leader')
|
||||
assert len(r) == 1
|
||||
assert r[0].name == 'leader'
|
||||
|
||||
r = list(get_all_members(get_cluster_initialized_with_leader(), role='replica'))
|
||||
self.assertEquals(len(r), 1)
|
||||
self.assertEquals(r[0].name, 'other')
|
||||
assert len(r) == 1
|
||||
assert r[0].name == 'other'
|
||||
|
||||
self.assertEquals(len(list(get_all_members(get_cluster_initialized_without_leader(), role='replica'))), 2)
|
||||
r = list(get_all_members(get_cluster_initialized_without_leader(), role='replica'))
|
||||
assert len(r) == 2
|
||||
|
||||
@patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
|
||||
@patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
|
||||
@patch('requests.get', requests_get)
|
||||
@patch('requests.post', requests_get)
|
||||
def test_members(self):
|
||||
result = self.runner.invoke(members, ['alpha'])
|
||||
runner = CliRunner()
|
||||
|
||||
result = runner.invoke(members, ['alpha'])
|
||||
assert result.exit_code == 0
|
||||
|
||||
def test_configure(self):
|
||||
result = self.runner.invoke(configure, [
|
||||
runner = CliRunner()
|
||||
|
||||
result = runner.invoke(configure, [
|
||||
'--dcs',
|
||||
'abc',
|
||||
'-c',
|
||||
@@ -388,3 +372,5 @@ leader''')
|
||||
])
|
||||
|
||||
assert result.exit_code == 0
|
||||
|
||||
|
||||
|
||||
+32
-27
@@ -7,12 +7,11 @@ import unittest
|
||||
|
||||
from dns.exception import DNSException
|
||||
from mock import Mock, patch
|
||||
from patroni.dcs import Cluster
|
||||
from patroni.dcs import Cluster, DCSError, Leader
|
||||
from patroni.etcd import Client, Etcd, EtcdError
|
||||
from patroni.exceptions import DCSError
|
||||
|
||||
|
||||
class MockResponse(object):
|
||||
class MockResponse:
|
||||
|
||||
def __init__(self):
|
||||
self.status_code = 200
|
||||
@@ -25,24 +24,23 @@ class MockResponse(object):
|
||||
|
||||
@property
|
||||
def data(self):
|
||||
return self.content.encode('utf-8')
|
||||
if self.content == 'TimeoutError':
|
||||
raise urllib3.exceptions.TimeoutError
|
||||
if self.content == 'Exception':
|
||||
raise Exception
|
||||
return self.content
|
||||
|
||||
@property
|
||||
def status(self):
|
||||
return self.status_code
|
||||
|
||||
@staticmethod
|
||||
def getheader(*args):
|
||||
return ''
|
||||
|
||||
|
||||
class MockPostgresql(Mock):
|
||||
|
||||
server_version = '999999'
|
||||
scope = 'dummy'
|
||||
|
||||
@staticmethod
|
||||
def last_operation():
|
||||
def last_operation(self):
|
||||
return '0'
|
||||
|
||||
|
||||
@@ -55,7 +53,10 @@ def requests_get(url, **kwargs):
|
||||
elif ':8011/patroni' in url:
|
||||
response.content = '{"role": "replica", "xlog": {"replayed_location": 0}, "tags": {}}'
|
||||
elif url.endswith('/members'):
|
||||
response.content = '[{}]' if url.startswith('http://error') else members
|
||||
if url.startswith('http://error'):
|
||||
response.content = '[{}]'
|
||||
else:
|
||||
response.content = members
|
||||
elif url.startswith('http://exhibitor'):
|
||||
response.content = '{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}'
|
||||
else:
|
||||
@@ -66,7 +67,7 @@ def requests_get(url, **kwargs):
|
||||
|
||||
def etcd_watch(key, index=None, timeout=None, recursive=None):
|
||||
if timeout == 2.0:
|
||||
raise etcd.EtcdWatchTimedOut
|
||||
raise urllib3.exceptions.TimeoutError
|
||||
elif timeout == 5.0:
|
||||
return etcd.EtcdResult('delete', {})
|
||||
elif timeout == 10.0:
|
||||
@@ -80,9 +81,9 @@ def etcd_watch(key, index=None, timeout=None, recursive=None):
|
||||
def etcd_write(key, value, **kwargs):
|
||||
if key == '/service/exists/leader':
|
||||
raise etcd.EtcdAlreadyExist
|
||||
if key in ['/service/test/leader', '/patroni/test/leader'] and \
|
||||
(kwargs.get('prevValue') == 'foo' or not kwargs.get('prevExist', True)):
|
||||
return True
|
||||
if key == '/service/test/leader' or key == '/patroni/test/leader':
|
||||
if kwargs.get('prevValue', None) == 'foo' or not kwargs.get('prevExist', True):
|
||||
return True
|
||||
raise etcd.EtcdException
|
||||
|
||||
|
||||
@@ -123,12 +124,12 @@ class SleepException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class MockSRV(object):
|
||||
class MockSRV:
|
||||
port = 2380
|
||||
target = '127.0.0.1'
|
||||
|
||||
|
||||
def dns_query(name, _):
|
||||
def dns_query(name, type):
|
||||
if name == '_etcd-server._tcp.blabla':
|
||||
return []
|
||||
elif name == '_etcd-server._tcp.exception':
|
||||
@@ -143,8 +144,6 @@ def socket_getaddrinfo(*args):
|
||||
|
||||
|
||||
def http_request(method, url, **kwargs):
|
||||
if url == 'http://localhost:2379/timeout':
|
||||
raise urllib3.exceptions.ReadTimeoutError(None, None, None)
|
||||
if url == 'http://localhost:2379/':
|
||||
return MockResponse()
|
||||
raise socket.error
|
||||
@@ -162,27 +161,30 @@ class TestClient(unittest.TestCase):
|
||||
mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
|
||||
self.client = Client({'discovery_srv': 'test'})
|
||||
self.client.http.request = http_request
|
||||
self.client.http.request_encode_body = http_request
|
||||
|
||||
def test_api_execute(self):
|
||||
self.client._base_uri = 'http://localhost:4001'
|
||||
self.client._machines_cache = ['http://localhost:2379']
|
||||
self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
|
||||
self.client._update_machines_cache = False
|
||||
self.client.api_execute('/', 'POST', timeout=0)
|
||||
self.client.api_execute('/', 'GET')
|
||||
self.client._update_machines_cache = False
|
||||
self.client._base_uri = 'http://localhost:4001'
|
||||
self.client._machines_cache = []
|
||||
self.assertRaises(etcd.EtcdConnectionFailed, self.client.api_execute, '/', 'GET')
|
||||
self.assertTrue(self.client._update_machines_cache)
|
||||
self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'GET')
|
||||
self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
|
||||
self.assertRaises(ValueError, self.client.api_execute, '', '')
|
||||
|
||||
def test_get_srv_record(self):
|
||||
self.assertEquals(self.client.get_srv_record('blabla'), [])
|
||||
self.assertEquals(self.client.get_srv_record('exception'), [])
|
||||
|
||||
def test__result_from_response(self):
|
||||
response = MockResponse()
|
||||
response.content = 'TimeoutError'
|
||||
self.assertRaises(urllib3.exceptions.TimeoutError, self.client._result_from_response, response)
|
||||
response.content = 'Exception'
|
||||
self.assertRaises(etcd.EtcdException, self.client._result_from_response, response)
|
||||
response.content = b'{}'
|
||||
self.assertRaises(etcd.EtcdException, self.client._result_from_response, response)
|
||||
|
||||
def test__get_machines_cache_from_srv(self):
|
||||
self.client.get_srv_record = Mock(return_value=[('localhost', 2380)])
|
||||
self.client._get_machines_cache_from_srv('blabla')
|
||||
@@ -224,8 +226,11 @@ class TestEtcd(unittest.TestCase):
|
||||
cluster = self.etcd.get_cluster()
|
||||
self.assertIsInstance(cluster, Cluster)
|
||||
self.assertIsNone(cluster.leader)
|
||||
|
||||
def test_current_leader(self):
|
||||
self.assertIsInstance(self.etcd.current_leader(), Leader)
|
||||
self.etcd._base_path = '/service/noleader'
|
||||
self.assertRaises(EtcdError, self.etcd.get_cluster)
|
||||
self.assertIsNone(self.etcd.current_leader())
|
||||
|
||||
def test_touch_member(self):
|
||||
self.assertFalse(self.etcd.touch_member('', ''))
|
||||
|
||||
+76
-114
@@ -1,14 +1,11 @@
|
||||
import etcd
|
||||
import unittest
|
||||
import datetime
|
||||
import pytz
|
||||
|
||||
from etcd import EtcdException
|
||||
from mock import Mock, MagicMock, patch
|
||||
from patroni.dcs import Cluster, Failover, Leader, Member
|
||||
from patroni.etcd import Client, Etcd
|
||||
from patroni.exceptions import DCSError, PostgresException
|
||||
from patroni.ha import Ha
|
||||
from patroni.postgresql import Postgresql
|
||||
from test_etcd import socket_getaddrinfo, etcd_read, etcd_write, requests_get
|
||||
|
||||
|
||||
@@ -21,7 +18,7 @@ def false(*args, **kwargs):
|
||||
|
||||
|
||||
def get_cluster(initialize, leader, members, failover):
|
||||
return Cluster(initialize, leader, 10, members, failover)
|
||||
return Cluster(initialize, leader, None, members, failover)
|
||||
|
||||
|
||||
def get_cluster_not_initialized_without_leader():
|
||||
@@ -29,24 +26,61 @@ def get_cluster_not_initialized_without_leader():
|
||||
|
||||
|
||||
def get_cluster_initialized_without_leader(leader=False, failover=None):
|
||||
m1 = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres',
|
||||
'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location': 4})
|
||||
l = Leader(0, 0, m1) if leader else None
|
||||
m2 = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
|
||||
'api_url': 'http://127.0.0.1:8011/patroni'})
|
||||
return get_cluster(True, l, [m1, m2], failover)
|
||||
m = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres',
|
||||
'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location':4})
|
||||
l = Leader(0, 0, m) if leader else None
|
||||
o = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
|
||||
'api_url': 'http://127.0.0.1:8011/patroni'})
|
||||
return get_cluster(True, l, [m, o], failover)
|
||||
|
||||
|
||||
def get_cluster_initialized_with_leader(failover=None):
|
||||
return get_cluster_initialized_without_leader(leader=True, failover=failover)
|
||||
|
||||
|
||||
def get_cluster_initialized_with_only_leader(failover=None):
|
||||
l = get_cluster_initialized_without_leader(leader=True, failover=failover).leader
|
||||
return get_cluster(True, l, [l], failover)
|
||||
|
||||
|
||||
class MockPatroni(object):
|
||||
class MockPostgresql(Mock):
|
||||
|
||||
name = 'postgresql0'
|
||||
role = 'replica'
|
||||
state = 'running'
|
||||
connection_string = 'postgres://foo@bar/postgres'
|
||||
|
||||
def is_healthy(self):
|
||||
return True
|
||||
|
||||
def start(self):
|
||||
return True
|
||||
|
||||
def is_healthiest_node(self, members):
|
||||
return True
|
||||
|
||||
def is_leader(self):
|
||||
return True
|
||||
|
||||
def xlog_position(self):
|
||||
return 0
|
||||
|
||||
def last_operation(self):
|
||||
return 0
|
||||
|
||||
def data_directory_empty(self):
|
||||
return False
|
||||
|
||||
def bootstrap(self, *args, **kwargs):
|
||||
return True
|
||||
|
||||
def check_replication_lag(self, last_leader_operation):
|
||||
return True
|
||||
|
||||
def check_recovery_conf(self, leader):
|
||||
return False
|
||||
|
||||
|
||||
class MockPatroni:
|
||||
|
||||
def __init__(self, p, d):
|
||||
self.postgresql = p
|
||||
@@ -54,50 +88,30 @@ class MockPatroni(object):
|
||||
self.api = Mock()
|
||||
self.tags = {}
|
||||
self.nofailover = None
|
||||
self.nap_time = 10
|
||||
self.replicatefrom = None
|
||||
self.bdr = {}
|
||||
self.api.connection_string = 'http://127.0.0.1:8008'
|
||||
self.clonefrom = None
|
||||
|
||||
|
||||
def run_async(func, args=()):
|
||||
return func(*args) if args else func()
|
||||
func(*args) if args else func()
|
||||
|
||||
|
||||
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
|
||||
@patch.object(Postgresql, 'is_leader', Mock(return_value=True))
|
||||
@patch.object(Postgresql, 'xlog_position', Mock(return_value=0))
|
||||
@patch.object(Postgresql, 'call_nowait', Mock(return_value=True))
|
||||
@patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
|
||||
@patch.object(Postgresql, 'controldata', Mock(return_value={'Database system identifier': '1234567890'}))
|
||||
@patch.object(Postgresql, 'sync_replication_slots', Mock())
|
||||
@patch.object(Postgresql, 'write_pg_hba', Mock())
|
||||
@patch.object(Postgresql, 'write_pgpass', Mock())
|
||||
@patch.object(Postgresql, 'write_recovery_conf', Mock())
|
||||
@patch.object(Postgresql, 'query', Mock())
|
||||
@patch.object(Postgresql, 'checkpoint', Mock())
|
||||
@patch('subprocess.call', Mock(return_value=0))
|
||||
class TestHa(unittest.TestCase):
|
||||
|
||||
@patch('socket.getaddrinfo', socket_getaddrinfo)
|
||||
def setUp(self):
|
||||
with patch.object(Client, 'machines') as mock_machines:
|
||||
mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
|
||||
self.p = Postgresql({'name': 'postgresql0', 'scope': 'dummy', 'listen': '127.0.0.1:5432',
|
||||
'data_dir': 'data/postgresql0', 'superuser': {}, 'admin': {},
|
||||
'replication': {'username': '', 'password': '', 'network': ''}})
|
||||
self.p.set_state('running')
|
||||
self.p.check_replication_lag = true
|
||||
self.p.can_create_replica_without_replication_connection = MagicMock(return_value=False)
|
||||
self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
|
||||
self.e.client.read = etcd_read
|
||||
self.e.client.write = etcd_write
|
||||
self.e.client.delete = Mock(side_effect=EtcdException())
|
||||
self.ha = Ha(MockPatroni(self.p, self.e))
|
||||
self.ha._async_executor.run_async = run_async
|
||||
self.ha.old_cluster = self.e.get_cluster()
|
||||
self.ha.cluster = get_cluster_not_initialized_without_leader()
|
||||
self.ha.load_cluster_from_dcs = Mock()
|
||||
@patch.object(Client, 'machines')
|
||||
def setUp(self, mock_machines):
|
||||
mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
|
||||
self.p = MockPostgresql()
|
||||
self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
|
||||
self.e.client.read = etcd_read
|
||||
self.e.client.write = etcd_write
|
||||
self.e.client.delete = Mock(side_effect=etcd.EtcdException())
|
||||
self.ha = Ha(MockPatroni(self.p, self.e))
|
||||
self.ha._async_executor.run_async = run_async
|
||||
self.ha.old_cluster = self.e.get_cluster()
|
||||
self.ha.cluster = get_cluster_not_initialized_without_leader()
|
||||
self.ha.load_cluster_from_dcs = Mock()
|
||||
|
||||
def test_update_lock(self):
|
||||
self.p.last_operation = Mock(side_effect=PostgresException(''))
|
||||
@@ -114,19 +128,13 @@ class TestHa(unittest.TestCase):
|
||||
def test_recover_replica_failed(self):
|
||||
self.p.controldata = lambda: {'Database cluster state': 'in production'}
|
||||
self.p.is_healthy = false
|
||||
self.p.is_running = false
|
||||
self.p.follow = false
|
||||
self.assertEquals(self.ha.run_cycle(), 'started as a secondary')
|
||||
self.p.follow_the_leader = false
|
||||
self.assertEquals(self.ha.run_cycle(), 'failed to start postgres')
|
||||
|
||||
def test_recover_master_failed(self):
|
||||
self.p.follow = false
|
||||
self.p.follow_the_leader = false
|
||||
self.p.is_healthy = false
|
||||
self.p.is_running = false
|
||||
self.ha.has_lock = true
|
||||
self.p.set_role('master')
|
||||
self.p.controldata = lambda: {'Database cluster state': 'in production'}
|
||||
self.assertEquals(self.ha.run_cycle(), 'started as readonly because i had the session lock')
|
||||
self.assertEquals(self.ha.run_cycle(), 'removed leader key after trying and failing to start postgres')
|
||||
|
||||
@patch('sys.exit', return_value=1)
|
||||
@@ -137,8 +145,7 @@ class TestHa(unittest.TestCase):
|
||||
|
||||
@patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
|
||||
def test_start_as_readonly(self):
|
||||
self.p.is_leader = false
|
||||
self.p.is_healthy = true
|
||||
self.p.is_leader = self.p.is_healthy = false
|
||||
self.ha.has_lock = true
|
||||
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader because i had the session lock')
|
||||
|
||||
@@ -152,7 +159,7 @@ class TestHa(unittest.TestCase):
|
||||
|
||||
def test_demote_after_failing_to_obtain_lock(self):
|
||||
self.ha.acquire_lock = false
|
||||
self.assertEquals(self.ha.run_cycle(), 'demoted self after trying and failing to obtain lock')
|
||||
self.assertEquals(self.ha.run_cycle(), 'demoted self due after trying and failing to obtain lock')
|
||||
|
||||
def test_follow_new_leader_after_failing_to_obtain_lock(self):
|
||||
self.ha.is_healthiest_node = true
|
||||
@@ -190,12 +197,10 @@ class TestHa(unittest.TestCase):
|
||||
self.ha.update_lock = false
|
||||
self.assertEquals(self.ha.run_cycle(), 'demoting self because i do not have the lock and i was a leader')
|
||||
|
||||
def test_follow(self):
|
||||
def test_follow_the_leader(self):
|
||||
self.ha.cluster.is_unlocked = false
|
||||
self.p.is_leader = false
|
||||
self.assertEquals(self.ha.run_cycle(), 'no action. i am a secondary and i am following a leader')
|
||||
self.ha.patroni.replicatefrom = "foo"
|
||||
self.assertEquals(self.ha.run_cycle(), 'no action. i am a secondary and i am following a leader')
|
||||
|
||||
def test_no_etcd_connection_master_demote(self):
|
||||
self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
|
||||
@@ -206,20 +211,10 @@ class TestHa(unittest.TestCase):
|
||||
self.p.bootstrap = false
|
||||
self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap from leader')
|
||||
|
||||
def test_bootstrap_from_another_member(self):
|
||||
self.ha.cluster = get_cluster_initialized_with_leader()
|
||||
self.ha.patroni.clonefrom = 'other'
|
||||
self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap from replica \'other\'')
|
||||
|
||||
def test_bootstrap_waiting_for_leader(self):
|
||||
self.ha.cluster = get_cluster_initialized_without_leader()
|
||||
self.assertEquals(self.ha.bootstrap(), 'waiting for leader to bootstrap')
|
||||
|
||||
def test_bootstrap_without_leader(self):
|
||||
self.ha.cluster = get_cluster_initialized_without_leader()
|
||||
self.p.can_create_replica_without_replication_connection = MagicMock(return_value=True)
|
||||
self.assertEquals(self.ha.bootstrap(), "trying to bootstrap without leader")
|
||||
|
||||
def test_bootstrap_initialize_lock_failed(self):
|
||||
self.ha.cluster = get_cluster_not_initialized_without_leader()
|
||||
self.assertEquals(self.ha.bootstrap(), 'failed to acquire initialize lock')
|
||||
@@ -275,62 +270,38 @@ class TestHa(unittest.TestCase):
|
||||
@patch('requests.get', requests_get)
|
||||
def test_manual_failover_from_leader(self):
|
||||
self.ha.has_lock = true
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None))
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', ''))
|
||||
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None))
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', MockPostgresql.name))
|
||||
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla', None))
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla'))
|
||||
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
|
||||
f = Failover(0, self.p.name, '', None)
|
||||
f = Failover(0, MockPostgresql.name, '')
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(f)
|
||||
self.assertEquals(self.ha.run_cycle(), 'manual failover: demoting myself')
|
||||
self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'})
|
||||
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
|
||||
# manual failover from the previous leader to us won't happen if we hold the nofailover flag
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, None))
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name))
|
||||
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
|
||||
|
||||
# Failover scheduled time must include timezone
|
||||
scheduled = datetime.datetime.now()
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.ha.run_cycle()
|
||||
|
||||
scheduled = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle())
|
||||
|
||||
scheduled = scheduled + datetime.timedelta(seconds=30)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle())
|
||||
|
||||
scheduled = scheduled + datetime.timedelta(seconds=-600)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle())
|
||||
|
||||
scheduled = None
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle())
|
||||
|
||||
@patch('requests.get', requests_get)
|
||||
def test_manual_failover_process_no_leader(self):
|
||||
self.p.is_leader = false
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', self.p.name, None))
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', MockPostgresql.name))
|
||||
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None))
|
||||
self.p.set_role('replica')
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader'))
|
||||
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
self.ha.fetch_node_status = lambda e: (e, True, True, 0, {}) # accessible, in_recovery
|
||||
self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, self.p.name, '', None))
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, MockPostgresql.name, ''))
|
||||
self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
self.ha.fetch_node_status = lambda e: (e, False, True, 0, {}) # inaccessible, in_recovery
|
||||
self.p.set_role('replica')
|
||||
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
# set failover flag to True for all members of the cluster
|
||||
# this should elect the current member, as we are not going to call the API for it.
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other'))
|
||||
self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'}) # accessible, in_recovery
|
||||
self.p.set_role('replica')
|
||||
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
# same as previous, but set the current member to nofailover. In no case it should be elected as a leader
|
||||
self.ha.patroni.nofailover = True
|
||||
@@ -363,12 +334,3 @@ class TestHa(unittest.TestCase):
|
||||
self.ha.fetch_node_status(member)
|
||||
member = Member(0, 'test', 1, {'api_url': 'http://localhost:8011/patroni'})
|
||||
self.ha.fetch_node_status(member)
|
||||
|
||||
def test_post_recover(self):
|
||||
self.p.is_running = false
|
||||
self.ha.has_lock = true
|
||||
self.assertEqual(self.ha.post_recover(), 'removed leader key after trying and failing to start postgres')
|
||||
self.ha.has_lock = false
|
||||
self.assertEqual(self.ha.post_recover(), 'failed to start postgres')
|
||||
self.p.is_running = true
|
||||
self.assertIsNone(self.ha.post_recover())
|
||||
|
||||
+22
-23
@@ -7,7 +7,7 @@ from mock import Mock, patch
|
||||
from patroni.api import RestApiServer
|
||||
from patroni.async_executor import AsyncExecutor
|
||||
from patroni.etcd import Etcd
|
||||
from patroni import Patroni, main as _main
|
||||
from patroni import Patroni, main
|
||||
from patroni.zookeeper import ZooKeeper
|
||||
from six.moves import BaseHTTPServer
|
||||
from test_etcd import Client, SleepException, etcd_read, etcd_write
|
||||
@@ -15,6 +15,10 @@ from test_postgresql import Postgresql, psycopg2_connect
|
||||
from test_zookeeper import MockKazooClient
|
||||
|
||||
|
||||
def time_sleep(*args):
|
||||
raise SleepException()
|
||||
|
||||
|
||||
@patch('time.sleep', Mock())
|
||||
@patch('subprocess.call', Mock(return_value=0))
|
||||
@patch('psycopg2.connect', psycopg2_connect)
|
||||
@@ -24,19 +28,19 @@ from test_zookeeper import MockKazooClient
|
||||
@patch.object(AsyncExecutor, 'run', Mock())
|
||||
class TestPatroni(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
with patch.object(Client, 'machines') as mock_machines:
|
||||
mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
|
||||
self.touched = False
|
||||
self.init_cancelled = False
|
||||
RestApiServer._BaseServer__is_shut_down = Mock()
|
||||
RestApiServer._BaseServer__shutdown_request = True
|
||||
RestApiServer.socket = 0
|
||||
with open('postgres0.yml', 'r') as f:
|
||||
config = yaml.load(f)
|
||||
self.p = Patroni(config)
|
||||
self.p.ha.dcs.client.write = etcd_write
|
||||
self.p.ha.dcs.client.read = etcd_read
|
||||
@patch.object(Client, 'machines')
|
||||
def setUp(self, mock_machines):
|
||||
mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
|
||||
self.touched = False
|
||||
self.init_cancelled = False
|
||||
RestApiServer._BaseServer__is_shut_down = Mock()
|
||||
RestApiServer._BaseServer__shutdown_request = True
|
||||
RestApiServer.socket = 0
|
||||
with open('postgres0.yml', 'r') as f:
|
||||
config = yaml.load(f)
|
||||
self.p = Patroni(config)
|
||||
self.p.ha.dcs.client.write = etcd_write
|
||||
self.p.ha.dcs.client.read = etcd_read
|
||||
|
||||
@patch('patroni.zookeeper.KazooClient', MockKazooClient())
|
||||
def test_get_dcs(self):
|
||||
@@ -47,18 +51,18 @@ class TestPatroni(unittest.TestCase):
|
||||
@patch.object(Etcd, 'delete_leader', Mock())
|
||||
@patch.object(Client, 'machines')
|
||||
def test_patroni_main(self, mock_machines):
|
||||
_main()
|
||||
main()
|
||||
sys.argv = ['patroni.py', 'postgres0.yml']
|
||||
|
||||
mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
|
||||
with patch.object(Patroni, 'run', Mock(side_effect=SleepException())):
|
||||
self.assertRaises(SleepException, _main)
|
||||
self.assertRaises(SleepException, main)
|
||||
with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())):
|
||||
_main()
|
||||
main()
|
||||
|
||||
@patch('time.sleep', Mock(side_effect=SleepException()))
|
||||
def test_run(self):
|
||||
self.p.ha.dcs.watch = Mock(side_effect=SleepException())
|
||||
self.p.ha.dcs.watch = time_sleep
|
||||
self.assertRaises(SleepException, self.p.run)
|
||||
|
||||
self.p.ha.state_handler.is_leader = Mock(return_value=False)
|
||||
@@ -76,8 +80,3 @@ class TestPatroni(unittest.TestCase):
|
||||
self.assertTrue(self.p.nofailover)
|
||||
self.p.tags['nofailover'] = None
|
||||
self.assertFalse(self.p.nofailover)
|
||||
|
||||
def test_replicatefrom(self):
|
||||
self.assertIsNone(self.p.replicatefrom)
|
||||
self.p.tags['replicatefrom'] = 'foo'
|
||||
self.assertEqual(self.p.replicatefrom, 'foo')
|
||||
|
||||
+76
-81
@@ -2,19 +2,24 @@ import mock # for the mock.call method, importing it without a namespace breaks
|
||||
import os
|
||||
import psycopg2
|
||||
import shutil
|
||||
import subprocess
|
||||
import unittest
|
||||
|
||||
from six.moves import builtins
|
||||
from mock import Mock, MagicMock, PropertyMock, patch, mock_open
|
||||
from patroni.dcs import Cluster, Leader, Member
|
||||
from patroni.exceptions import PostgresException, PostgresConnectionException
|
||||
from patroni.postgresql import Postgresql
|
||||
from patroni.utils import RetryFailedError
|
||||
from six.moves import builtins
|
||||
from test_ha import false
|
||||
import subprocess
|
||||
|
||||
|
||||
class MockCursor(object):
|
||||
def is_file_raise_on_backup(*args, **kwargs):
|
||||
if args[0].endswith('.backup'):
|
||||
raise Exception("foo")
|
||||
|
||||
|
||||
class MockCursor:
|
||||
|
||||
def __init__(self, connection):
|
||||
self.connection = connection
|
||||
@@ -33,7 +38,7 @@ class MockCursor(object):
|
||||
elif sql == 'SELECT pg_is_in_recovery()':
|
||||
self.results = [(False, )]
|
||||
elif sql.startswith('SELECT to_char(pg_postmaster_start_time'):
|
||||
self.results = [('', True, '', '', '', '', False)]
|
||||
self.results = [('', True, '', '', '', False)]
|
||||
else:
|
||||
self.results = [(
|
||||
None,
|
||||
@@ -54,8 +59,7 @@ class MockCursor(object):
|
||||
def fetchall(self):
|
||||
return self.results
|
||||
|
||||
@staticmethod
|
||||
def close():
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
def __iter__(self):
|
||||
@@ -150,12 +154,9 @@ def psycopg2_connect(*args, **kwargs):
|
||||
return MockConnect()
|
||||
|
||||
|
||||
def fake_listdir(path):
|
||||
return ["a", "b", "c"] if path.endswith('pg_xlog/archive_status') else []
|
||||
|
||||
|
||||
@patch('subprocess.call', Mock(return_value=0))
|
||||
@patch('psycopg2.connect', psycopg2_connect)
|
||||
@patch('shutil.copy', Mock())
|
||||
class TestPostgresql(unittest.TestCase):
|
||||
|
||||
@patch('subprocess.call', Mock(return_value=0))
|
||||
@@ -164,7 +165,7 @@ class TestPostgresql(unittest.TestCase):
|
||||
self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': 'data/test0',
|
||||
'listen': '127.0.0.1, *:5432', 'connect_address': '127.0.0.2:5432',
|
||||
'pg_hba': ['hostssl all all 0.0.0.0/0 md5', 'host all all 0.0.0.0/0 md5'],
|
||||
'superuser': {'username': 'test', 'password': 'test'},
|
||||
'superuser': {'password': 'test'},
|
||||
'admin': {'username': 'admin', 'password': 'admin'},
|
||||
'pg_rewind': {'username': 'admin', 'password': 'admin'},
|
||||
'replication': {'username': 'replicator',
|
||||
@@ -180,8 +181,7 @@ class TestPostgresql(unittest.TestCase):
|
||||
os.makedirs(self.p.data_dir)
|
||||
self.leadermem = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres'})
|
||||
self.leader = Leader(-1, 28, self.leadermem)
|
||||
self.other = Member(0, 'test1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres',
|
||||
'tags': {'replicatefrom': 'leader'}})
|
||||
self.other = Member(0, 'test1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres'})
|
||||
self.me = Member(0, 'test0', 28, {'conn_url': 'postgres://replicator:[email protected]:5434/postgres'})
|
||||
|
||||
def tearDown(self):
|
||||
@@ -204,11 +204,6 @@ class TestPostgresql(unittest.TestCase):
|
||||
self.assertTrue(self.p.initialize())
|
||||
self.assertTrue(os.path.exists(os.path.join(self.p.data_dir, 'pg_hba.conf')))
|
||||
|
||||
@patch('os.path.exists', Mock(return_value=True))
|
||||
@patch('os.unlink', Mock())
|
||||
def test_delete_trigger_file(self):
|
||||
self.p.delete_trigger_file()
|
||||
|
||||
def test_start(self):
|
||||
self.assertTrue(self.p.start())
|
||||
self.p.is_running = false
|
||||
@@ -233,12 +228,10 @@ class TestPostgresql(unittest.TestCase):
|
||||
self.p.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo', 'password': 'bar'})
|
||||
|
||||
@patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
|
||||
def test_sync_replica(self):
|
||||
self.assertTrue(self.p.sync_replica(self.leader))
|
||||
self.p.create_replica = Mock(return_value=1)
|
||||
self.assertFalse(self.p.sync_replica(self.leader))
|
||||
def test_sync_from_leader(self):
|
||||
self.assertTrue(self.p.sync_from_leader(self.leader))
|
||||
|
||||
@patch('subprocess.call', side_effect=OSError)
|
||||
@patch('subprocess.call', side_effect=Exception("Test"))
|
||||
@patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
|
||||
def test_pg_rewind(self, mock_call):
|
||||
self.assertTrue(self.p.rewind(self.leader))
|
||||
@@ -249,28 +242,26 @@ class TestPostgresql(unittest.TestCase):
|
||||
@patch('patroni.postgresql.Postgresql.remove_data_directory', MagicMock(return_value=True))
|
||||
@patch('patroni.postgresql.Postgresql.single_user_mode', MagicMock(return_value=1))
|
||||
@patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
|
||||
@patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string))
|
||||
def test_follow(self, mock_pg_rewind):
|
||||
self.p.follow(None)
|
||||
self.p.follow(self.leader)
|
||||
self.p.follow(Leader(-1, 28, self.other))
|
||||
def test_follow_the_leader(self, mock_pg_rewind):
|
||||
self.p.demote()
|
||||
self.p.follow_the_leader(None)
|
||||
self.p.demote()
|
||||
self.p.follow_the_leader(self.leader)
|
||||
self.p.follow_the_leader(Leader(-1, 28, self.other))
|
||||
self.p.rewind = mock_pg_rewind
|
||||
self.p.follow(self.leader)
|
||||
self.p.follow_the_leader(self.leader)
|
||||
self.p.require_rewind()
|
||||
with mock.patch('os.path.islink', MagicMock(return_value=True)):
|
||||
with mock.patch('patroni.postgresql.Postgresql.can_rewind', new_callable=PropertyMock(return_value=True)):
|
||||
with mock.patch('os.unlink', MagicMock(return_value=True)):
|
||||
self.p.follow(self.leader, recovery=True)
|
||||
self.p.follow_the_leader(self.leader, recovery=True)
|
||||
self.p.require_rewind()
|
||||
with mock.patch('patroni.postgresql.Postgresql.can_rewind', new_callable=PropertyMock(return_value=True)):
|
||||
self.p.rewind.return_value = True
|
||||
self.p.follow(self.leader, recovery=True)
|
||||
self.p.follow_the_leader(self.leader, recovery=True)
|
||||
self.p.rewind.return_value = False
|
||||
self.p.follow(self.leader, recovery=True)
|
||||
with mock.patch('patroni.postgresql.Postgresql.check_recovery_conf', MagicMock(return_value=True)):
|
||||
self.assertTrue(self.p.follow(None))
|
||||
self.p.follow_the_leader(self.leader, recovery=True)
|
||||
|
||||
@patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string))
|
||||
def test_can_rewind(self):
|
||||
tmp = self.p.pg_rewind
|
||||
self.p.pg_rewind = None
|
||||
@@ -278,16 +269,16 @@ class TestPostgresql(unittest.TestCase):
|
||||
self.p.pg_rewind = tmp
|
||||
with mock.patch('subprocess.call', MagicMock(return_value=1)):
|
||||
self.assertFalse(self.p.can_rewind)
|
||||
with mock.patch('subprocess.call', side_effect=OSError):
|
||||
with mock.patch('subprocess.call', side_effect=OSError("foo")):
|
||||
self.assertFalse(self.p.can_rewind)
|
||||
tmp = self.p.controldata
|
||||
tmp = self.p.controldata()
|
||||
self.p.controldata = lambda: {'wal_log_hints setting': 'on'}
|
||||
self.assertTrue(self.p.can_rewind)
|
||||
self.p.controldata = tmp
|
||||
|
||||
@patch('time.sleep', Mock())
|
||||
def test_create_replica(self):
|
||||
self.p.delete_trigger_file = Mock(side_effect=OSError)
|
||||
self.p.delete_trigger_file = Mock(side_effect=OSError())
|
||||
with patch('subprocess.call', Mock(side_effect=[1, 0])):
|
||||
self.assertEquals(self.p.create_replica(self.leader, ''), 0)
|
||||
with patch('subprocess.call', Mock(side_effect=[Exception(), 0])):
|
||||
@@ -303,6 +294,12 @@ class TestPostgresql(unittest.TestCase):
|
||||
with patch('subprocess.call', Mock(side_effect=Exception("foo"))):
|
||||
self.assertEquals(self.p.create_replica(self.leader, ''), 1)
|
||||
|
||||
def test_create_connection_users(self):
|
||||
cfg = self.p.config
|
||||
cfg['superuser']['username'] = 'test'
|
||||
p = Postgresql(cfg)
|
||||
p.create_connection_users()
|
||||
|
||||
def test_sync_replication_slots(self):
|
||||
self.p.start()
|
||||
cluster = Cluster(True, self.leader, 0, [self.me, self.other, self.leadermem], None)
|
||||
@@ -310,15 +307,12 @@ class TestPostgresql(unittest.TestCase):
|
||||
self.p.query = Mock(side_effect=psycopg2.OperationalError)
|
||||
self.p.schedule_load_slots = True
|
||||
self.p.sync_replication_slots(cluster)
|
||||
self.p.schedule_load_slots = False
|
||||
with mock.patch('patroni.postgresql.Postgresql.role', new_callable=PropertyMock(return_value='replica')):
|
||||
self.p.sync_replication_slots(cluster)
|
||||
|
||||
@patch.object(MockConnect, 'closed', 2)
|
||||
def test__query(self):
|
||||
self.assertRaises(PostgresConnectionException, self.p._query, 'blabla')
|
||||
self.assertRaises(PostgresConnectionException, self.p._query, 'blabla', 'postgres')
|
||||
self.p._state = 'restarting'
|
||||
self.assertRaises(RetryFailedError, self.p._query, 'blabla')
|
||||
self.assertRaises(RetryFailedError, self.p._query, 'blabla', 'postgres')
|
||||
|
||||
def test_query(self):
|
||||
self.p.query('select 1')
|
||||
@@ -344,7 +338,7 @@ class TestPostgresql(unittest.TestCase):
|
||||
def test_last_operation(self):
|
||||
self.assertEquals(self.p.last_operation(), '0')
|
||||
|
||||
@patch('subprocess.Popen', Mock(side_effect=OSError))
|
||||
@patch('subprocess.Popen', Mock(side_effect=OSError()))
|
||||
def test_call_nowait(self):
|
||||
self.assertFalse(self.p.call_nowait('on_start'))
|
||||
|
||||
@@ -364,7 +358,7 @@ class TestPostgresql(unittest.TestCase):
|
||||
def test_move_data_directory(self):
|
||||
self.p.is_running = false
|
||||
self.p.move_data_directory()
|
||||
with patch('os.rename', Mock(side_effect=OSError)):
|
||||
with patch('os.rename', Mock(side_effect=OSError())):
|
||||
self.p.move_data_directory()
|
||||
|
||||
@patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
|
||||
@@ -372,8 +366,7 @@ class TestPostgresql(unittest.TestCase):
|
||||
with patch('subprocess.call', Mock(return_value=1)):
|
||||
self.assertRaises(PostgresException, self.p.bootstrap)
|
||||
self.p.bootstrap()
|
||||
with patch('patroni.postgresql.Postgresql.sync_replica', MagicMock(return_value=True)):
|
||||
self.p.bootstrap(self.leader)
|
||||
self.p.bootstrap(self.leader)
|
||||
|
||||
def test_remove_data_directory(self):
|
||||
self.p.data_dir = 'data_dir'
|
||||
@@ -383,20 +376,26 @@ class TestPostgresql(unittest.TestCase):
|
||||
open(self.p.data_dir, 'w').close()
|
||||
self.p.remove_data_directory()
|
||||
os.symlink('unexisting', self.p.data_dir)
|
||||
with patch('os.unlink', Mock(side_effect=OSError)):
|
||||
with patch('os.unlink', Mock(side_effect=Exception)):
|
||||
self.p.remove_data_directory()
|
||||
self.p.remove_data_directory()
|
||||
|
||||
def test_controldata(self):
|
||||
with patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string)):
|
||||
data = self.p.controldata()
|
||||
self.assertEquals(len(data), 50)
|
||||
self.assertEquals(data['Database cluster state'], 'shut down in recovery')
|
||||
self.assertEquals(data['wal_log_hints setting'], 'on')
|
||||
self.assertEquals(int(data['Database block size']), 8192)
|
||||
@patch('subprocess.check_output', MagicMock(return_value=0, side_effect=pg_controldata_string))
|
||||
@patch('subprocess.check_output', side_effect=subprocess.CalledProcessError)
|
||||
@patch('subprocess.check_output', side_effect=Exception('Failed'))
|
||||
def test_controldata(self, check_output_call_error, check_output_generic_exception):
|
||||
data = self.p.controldata()
|
||||
self.assertEquals(len(data), 50)
|
||||
self.assertEquals(data['Database cluster state'], 'shut down in recovery')
|
||||
self.assertEquals(data['wal_log_hints setting'], 'on')
|
||||
self.assertEquals(int(data['Database block size']), 8192)
|
||||
|
||||
with patch('subprocess.check_output', Mock(side_effect=subprocess.CalledProcessError(1, ''))):
|
||||
self.assertEquals(self.p.controldata(), {})
|
||||
subprocess.check_output = check_output_call_error
|
||||
data = self.p.controldata()
|
||||
self.assertEquals(data, dict())
|
||||
|
||||
subprocess.check_output = check_output_generic_exception
|
||||
self.assertRaises(Exception, self.p.controldata())
|
||||
|
||||
def test_read_postmaster_opts(self):
|
||||
m = mock_open(read_data=postmaster_opts_string())
|
||||
@@ -406,10 +405,13 @@ class TestPostgresql(unittest.TestCase):
|
||||
self.assertEquals(int(data['max_replication_slots']), 5)
|
||||
self.assertEqual(data.get('D'), None)
|
||||
|
||||
m.side_effect = IOError
|
||||
m.side_effect = IOError("foo")
|
||||
data = self.p.read_postmaster_opts()
|
||||
self.assertEqual(data, dict())
|
||||
|
||||
m.side_effect = Exception("foo")
|
||||
self.assertRaises(Exception, self.p.read_postmaster_opts())
|
||||
|
||||
@patch('subprocess.Popen')
|
||||
@patch.object(builtins, 'open', MagicMock(return_value=42))
|
||||
def test_single_user_mode(self, subprocess_popen_mock):
|
||||
@@ -429,7 +431,13 @@ class TestPostgresql(unittest.TestCase):
|
||||
subprocess_popen_mock.return_value = None
|
||||
self.assertEquals(self.p.single_user_mode(), 1)
|
||||
|
||||
def fake_listdir(path):
|
||||
if path.endswith(os.path.join('pg_xlog', 'archive_status')):
|
||||
return ["a", "b", "c"]
|
||||
return []
|
||||
|
||||
@patch('os.listdir', MagicMock(side_effect=fake_listdir))
|
||||
@patch('os.path.isdir', MagicMock(return_value=True))
|
||||
@patch('os.unlink', return_value=True)
|
||||
@patch('os.remove', return_value=True)
|
||||
@patch('os.path.islink', return_value=False)
|
||||
@@ -451,8 +459,8 @@ class TestPostgresql(unittest.TestCase):
|
||||
mock_unlink.reset_mock()
|
||||
mock_remove.reset_mock()
|
||||
|
||||
mock_file.side_effect = OSError
|
||||
mock_link.side_effect = OSError
|
||||
mock_file.side_effect = Exception("foo")
|
||||
mock_link.side_effect = Exception("foo")
|
||||
self.p.cleanup_archive_status()
|
||||
mock_unlink.assert_not_called()
|
||||
mock_remove.assert_not_called()
|
||||
@@ -461,27 +469,14 @@ class TestPostgresql(unittest.TestCase):
|
||||
def test_sysid(self):
|
||||
self.assertEqual(self.p.sysid, "6200971513092291716")
|
||||
|
||||
@patch('os.path.isfile', Mock(return_value=True))
|
||||
@patch('shutil.copy', Mock(side_effect=IOError))
|
||||
def test_save_configuration_files(self):
|
||||
@patch('os.path.isfile', MagicMock(return_value=True))
|
||||
@patch('shutil.copy', side_effect=Exception)
|
||||
def test_save_configuration_files(self, mock_copy):
|
||||
shutil.copy = mock_copy
|
||||
self.p.save_configuration_files()
|
||||
|
||||
@patch('os.path.isfile', Mock(side_effect=[False, True]))
|
||||
@patch('shutil.copy', Mock(side_effect=IOError))
|
||||
def test_restore_configuration_files(self):
|
||||
@patch('os.path.isfile', MagicMock(side_effect=is_file_raise_on_backup))
|
||||
@patch('shutil.copy', side_effect=Exception)
|
||||
def test_restore_configuration_files(self, mock_copy):
|
||||
shutil.copy = mock_copy
|
||||
self.p.restore_configuration_files()
|
||||
|
||||
def test_can_create_replica_without_replication_connection(self):
|
||||
self.p.config['create_replica_method'] = []
|
||||
self.assertFalse(self.p.can_create_replica_without_replication_connection())
|
||||
self.p.config['create_replica_method'] = ['wale', 'basebackup']
|
||||
self.p.config['wale'] = {'command': 'foo', 'no_master': 1}
|
||||
self.assertTrue(self.p.can_create_replica_without_replication_connection())
|
||||
|
||||
def test_replica_method_can_work_without_replication_connection(self):
|
||||
self.assertFalse(self.p.replica_method_can_work_without_replication_connection('basebackup'))
|
||||
self.assertFalse(self.p.replica_method_can_work_without_replication_connection('foobar'))
|
||||
self.p.config['foo'] = {'command': 'bar', 'no_master': 1}
|
||||
self.assertTrue(self.p.replica_method_can_work_without_replication_connection('foo'))
|
||||
self.p.config['foo'] = {'command': 'bar'}
|
||||
self.assertFalse(self.p.replica_method_can_work_without_replication_connection('foo'))
|
||||
|
||||
+13
-11
@@ -16,21 +16,20 @@ class TestUtils(unittest.TestCase):
|
||||
|
||||
@patch('time.sleep', Mock())
|
||||
def test_reap_children(self):
|
||||
self.assertIsNone(reap_children())
|
||||
reap_children()
|
||||
with patch('os.waitpid', Mock(return_value=(0, 0))):
|
||||
sigchld_handler(None, None)
|
||||
self.assertIsNone(reap_children())
|
||||
reap_children()
|
||||
|
||||
@patch('time.sleep', time_sleep)
|
||||
def test_sleep(self):
|
||||
self.assertIsNone(sleep(0.01))
|
||||
sleep(0.01)
|
||||
|
||||
|
||||
@patch('time.sleep', Mock())
|
||||
class TestRetrySleeper(unittest.TestCase):
|
||||
|
||||
@staticmethod
|
||||
def _fail(times=1):
|
||||
def _fail(self, times=1):
|
||||
scope = dict(times=0)
|
||||
|
||||
def inner():
|
||||
@@ -41,33 +40,36 @@ class TestRetrySleeper(unittest.TestCase):
|
||||
raise PatroniException('Failed!')
|
||||
return inner
|
||||
|
||||
def _makeOne(self, *args, **kwargs):
|
||||
return Retry(*args, **kwargs)
|
||||
|
||||
def test_reset(self):
|
||||
retry = Retry(delay=0, max_tries=2)
|
||||
retry = self._makeOne(delay=0, max_tries=2)
|
||||
retry(self._fail())
|
||||
self.assertEquals(retry._attempts, 1)
|
||||
retry.reset()
|
||||
self.assertEquals(retry._attempts, 0)
|
||||
|
||||
def test_too_many_tries(self):
|
||||
retry = Retry(delay=0)
|
||||
retry = self._makeOne(delay=0)
|
||||
self.assertRaises(RetryFailedError, retry, self._fail(times=999))
|
||||
self.assertEquals(retry._attempts, 1)
|
||||
|
||||
def test_maximum_delay(self):
|
||||
retry = Retry(delay=10, max_tries=100)
|
||||
retry = self._makeOne(delay=10, max_tries=100)
|
||||
retry(self._fail(times=10))
|
||||
self.assertTrue(retry._cur_delay < 4000, retry._cur_delay)
|
||||
# gevent's sleep function is picky about the type
|
||||
self.assertEquals(type(retry._cur_delay), float)
|
||||
|
||||
def test_deadline(self):
|
||||
retry = Retry(deadline=0.0001)
|
||||
retry = self._makeOne(deadline=0.0001)
|
||||
self.assertRaises(RetryFailedError, retry, self._fail(times=100))
|
||||
|
||||
def test_copy(self):
|
||||
def _sleep(t):
|
||||
pass
|
||||
None
|
||||
|
||||
retry = Retry(sleep_func=_sleep)
|
||||
retry = self._makeOne(sleep_func=_sleep)
|
||||
rcopy = retry.copy()
|
||||
self.assertTrue(rcopy.sleep_func is _sleep)
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import unittest
|
||||
from mock import MagicMock, patch, PropertyMock
|
||||
import os
|
||||
import psycopg2
|
||||
import subprocess
|
||||
import unittest
|
||||
|
||||
from mock import MagicMock, patch, PropertyMock
|
||||
from patroni.scripts.wale_restore import WALERestore, main as _main
|
||||
from patroni.scripts.wale_restore import WALERestore
|
||||
|
||||
|
||||
def fake_cursor_fetchone(*args, **kwargs):
|
||||
@@ -28,19 +28,16 @@ def fake_backup_data(self, *args, **kwargs):
|
||||
base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 00000001000000000000007F 00000040 00000001000000000000007F 00000240
|
||||
"""
|
||||
|
||||
|
||||
def fake_backup_data_2(self, *args, **kwargs):
|
||||
""" return the fake result of WAL-E backup-list"""
|
||||
return """name last_modified expanded_size_bytes wal_segment_backup_start wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop """
|
||||
|
||||
|
||||
def fake_backup_data_3(self, *args, **kwargs):
|
||||
""" return the fake result of WAL-E backup-list"""
|
||||
return """name last_modified expanded_size_bytes wal_segment_backup_start wal_segment_offset_backup_start wal_segment_backup_stop
|
||||
base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 00000001000000000000007F 00000040 00000001000000000000007F 00000240
|
||||
"""
|
||||
|
||||
|
||||
def fake_backup_data_4(self, *args, **kwargs):
|
||||
""" return the fake result of WAL-E backup-list"""
|
||||
return """name last_modified expanded_size_foo wal_segment_backup_start wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop
|
||||
@@ -61,7 +58,7 @@ class TestWALERestore(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.wale_restore = WALERestore("batman", "/data",
|
||||
"host=batman port=5432 user=batman", "/etc", 100, 100, 1, 0)
|
||||
"host=batman port=5432 user=batman", "/etc", 100, 100, 1)
|
||||
|
||||
def tearDown(self):
|
||||
pass
|
||||
@@ -79,8 +76,6 @@ class TestWALERestore(unittest.TestCase):
|
||||
self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
|
||||
|
||||
self.wale_restore.should_use_s3_to_create_replica()
|
||||
self.wale_restore.no_master = 1
|
||||
self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
|
||||
|
||||
def test_create_replica_with_s3(self):
|
||||
with patch('subprocess.call', MagicMock(return_value=0)):
|
||||
@@ -94,8 +89,3 @@ class TestWALERestore(unittest.TestCase):
|
||||
with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', MagicMock(return_value=True)):
|
||||
with patch.object(self.wale_restore, 'create_replica_with_s3', MagicMock(return_value=0)):
|
||||
self.assertEqual(self.wale_restore.run(), 0)
|
||||
|
||||
@patch('sys.exit', MagicMock())
|
||||
@patch.object(WALERestore, 'run', MagicMock(return_value=0))
|
||||
def test_main(self):
|
||||
self.assertEqual(_main(), None)
|
||||
|
||||
@@ -20,8 +20,7 @@ class MockKazooClient(Mock):
|
||||
def client_id(self):
|
||||
return (-1, '')
|
||||
|
||||
@staticmethod
|
||||
def retry(func, *args, **kwargs):
|
||||
def retry(self, func, *args, **kwargs):
|
||||
func(*args, **kwargs)
|
||||
|
||||
def get(self, path, watch=None):
|
||||
@@ -44,8 +43,7 @@ class MockKazooClient(Mock):
|
||||
return (b'foo', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))
|
||||
return (b'', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))
|
||||
|
||||
@staticmethod
|
||||
def get_children(path, watch=None, include_data=False):
|
||||
def get_children(self, path, watch=None, include_data=False):
|
||||
if not isinstance(path, six.string_types):
|
||||
raise TypeError("Invalid type for 'path' (string expected)")
|
||||
if path.startswith('/no_node'):
|
||||
@@ -64,16 +62,16 @@ class MockKazooClient(Mock):
|
||||
elif value == b'retry' or (value == b'exists' and self.exists):
|
||||
raise NodeExistsError
|
||||
|
||||
@staticmethod
|
||||
def set(path, value, version=-1):
|
||||
def set(self, path, value, version=-1):
|
||||
if not isinstance(path, six.string_types):
|
||||
raise TypeError("Invalid type for 'path' (string expected)")
|
||||
if not isinstance(value, (six.binary_type,)):
|
||||
raise TypeError("Invalid type for 'value' (must be a byte string)")
|
||||
if path == '/service/bla/optime/leader':
|
||||
raise Exception
|
||||
if path == '/service/test/members/bar' and value == b'retry':
|
||||
return
|
||||
if path == '/service/test/members/bar':
|
||||
if value == b'retry':
|
||||
return
|
||||
if path == '/service/test/failover':
|
||||
if value == b'Exception':
|
||||
raise Exception
|
||||
|
||||
Reference in New Issue
Block a user