mirror of
https://github.com/outbackdingo/patroni.git
synced 2026-08-27 08:00:28 +00:00
Compare commits
2
Commits
v1.0
..
feature/bdr
| Author | SHA1 | Date | |
|---|---|---|---|
 Oleksii Kliukin
|
1eeb544431 | ||
|
Oleksii Kliukin
|
494565e6bb |
@@ -9,4 +9,3 @@ build/
|
||||
coverage.xml
|
||||
junit.xml
|
||||
pgpass
|
||||
scm-source.json
|
||||
|
||||
+8
-76
@@ -1,82 +1,14 @@
|
||||
sudo: false
|
||||
language: python
|
||||
python:
|
||||
- "3.5"
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
- postgresql-contrib-9.5
|
||||
postgresql: "9.5"
|
||||
env:
|
||||
global:
|
||||
- ETCDVERSION=2.3.2 ZKVERSION=3.4.6 CONSULVERSION=0.6.4
|
||||
matrix:
|
||||
- TEST_SUITE="python setup.py"
|
||||
- DCS="etcd" TEST_SUITE="behave"
|
||||
- DCS="exhibitor" TEST_SUITE="behave"
|
||||
- DCS="consul" TEST_SUITE="behave"
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/virtualenv/python2.7.9
|
||||
- $HOME/virtualenv/python3.4.2
|
||||
- $HOME/virtualenv/python3.5.0
|
||||
- "2.7"
|
||||
- "3.3"
|
||||
- "3.4"
|
||||
install:
|
||||
- |
|
||||
set -e
|
||||
|
||||
if [[ $TEST_SUITE == "behave" ]]; then
|
||||
if [[ $DCS == "consul" ]]; then
|
||||
curl -L https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip \
|
||||
| gunzip > consul
|
||||
chmod +x consul
|
||||
fi
|
||||
|
||||
if [[ $DCS == "etcd" ]]; then
|
||||
curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
|
||||
| tar xz -C . --strip=1 --wildcards --no-anchored etcd
|
||||
fi
|
||||
|
||||
if [[ $DCS == "exhibitor" ]]; then
|
||||
curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz
|
||||
mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg
|
||||
zookeeper-${ZKVERSION}/bin/zkServer.sh start
|
||||
# following lines are 'emulating' exhibitor REST API
|
||||
while true; do
|
||||
echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \
|
||||
| nc -l 8181 &> /dev/null
|
||||
done&
|
||||
fi
|
||||
fi
|
||||
|
||||
for pv in "2.7" "3.4" "3.5"; do
|
||||
source ~/virtualenv/python${pv}/bin/activate
|
||||
# explicitly install all needed python modules to cache them
|
||||
for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8 mock>=2.0.0 pytest-cov pytest'; do
|
||||
pip install $p
|
||||
done
|
||||
done
|
||||
- if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt --use-mirrors; fi
|
||||
- if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt --use-mirrors; fi
|
||||
- pip install coveralls
|
||||
script:
|
||||
- |
|
||||
for pv in "2.7" "3.4" "3.5"; do
|
||||
source ~/virtualenv/python${pv}/bin/activate
|
||||
|
||||
if [[ $TEST_SUITE == "behave" ]]; then
|
||||
if [[ $pv != "3.4" ]]; then
|
||||
echo Running acceptance tests using python${pv}
|
||||
if ! PATH=.:$PATH $TEST_SUITE; then
|
||||
# output all log files when tests are failing
|
||||
grep . features/output/*/*postgres?.*
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo Running unit tests using python${pv}
|
||||
$TEST_SUITE test
|
||||
$TEST_SUITE flake8
|
||||
fi
|
||||
done
|
||||
|
||||
set +e
|
||||
- python setup.py test
|
||||
- python setup.py flake8
|
||||
after_success:
|
||||
- coveralls
|
||||
- if [[ $TEST_SUITE != "behave" ]]; then python-codacy-coverage -r coverage.xml; fi
|
||||
|
||||
-12
@@ -1,12 +0,0 @@
|
||||
approvals:
|
||||
# PR needs at least 4 approvals
|
||||
minimum: 1
|
||||
# approval = comment that matches this regex
|
||||
pattern: "^\\s*(:?\\+1:?|👍)\\s*$"
|
||||
from:
|
||||
# commenter must be either one of:
|
||||
# a public zalando org member
|
||||
orgs:
|
||||
- zalando
|
||||
# a collaborator of the repo
|
||||
collaborators: true
|
||||
+29
-30
@@ -1,44 +1,43 @@
|
||||
## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
|
||||
## It has all the necessary components to play/debug with a single node appliance, running etcd
|
||||
FROM ubuntu:16.04
|
||||
FROM ubuntu:14.04
|
||||
MAINTAINER Feike Steenbergen <[email protected]>
|
||||
|
||||
RUN echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/01norecommend \
|
||||
&& echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/01norecommend
|
||||
# We need curl
|
||||
RUN apt-get update -y && apt-get install curl -y
|
||||
|
||||
# Add PGDG and BDR repositories
|
||||
RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
|
||||
RUN echo "deb http://packages.2ndquadrant.com/bdr/apt/ $(lsb_release -cs)-2ndquadrant main" >> /etc/apt/sources.list.d/pgdg.list
|
||||
RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
|
||||
# import the BDR key
|
||||
RUN curl -s -o - http://packages.2ndquadrant.com/bdr/apt/AA7A6805.asc | sudo apt-key add -
|
||||
|
||||
RUN apt-get update -y
|
||||
RUN apt-get upgrade -y
|
||||
|
||||
ENV PGVERSION 9.4
|
||||
ENV PGTYPE postgresql-bdr
|
||||
ENV PGCOMPATIBLETYPE postgresql
|
||||
|
||||
RUN apt-get install python python-yaml python-requests python-boto ${PGTYPE}-${PGVERSION} ${PGTYPE}-${PGVERSION}-bdr-plugin python-dnspython python-kazoo python-pip -y
|
||||
RUN apt-get install python-dev ${PGTYPE}-server-dev-${PGVERSION} -y
|
||||
RUN pip install python-etcd psycopg2
|
||||
|
||||
ENV PGVERSION 9.5
|
||||
ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH
|
||||
RUN apt-get update -y \
|
||||
&& apt-get upgrade -y \
|
||||
&& apt-get install -y curl jq haproxy zookeeper postgresql-${PGVERSION} python-psycopg2 python-yaml \
|
||||
python-requests python-six python-click python-dateutil python-tzlocal python-urllib3 \
|
||||
python-dnspython python-pip python-setuptools python-kazoo python-prettytable python \
|
||||
&& pip install python-etcd==0.4.3 python-consul==0.6.0 --upgrade \
|
||||
&& apt-get remove -y python-pip python-setuptools \
|
||||
&& apt-get autoremove -y \
|
||||
# Clean up
|
||||
&& apt-get clean -y \
|
||||
&& rm -rf /var/lib/apt/lists/* /root/.cache
|
||||
|
||||
ENV ETCDVERSION 2.3.6
|
||||
RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
|
||||
| tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl
|
||||
ADD patroni.py /patroni.py
|
||||
ADD patroni/ /patroni
|
||||
|
||||
ENV CONFDVERSION 0.11.0
|
||||
RUN curl -L https://github.com/kelseyhightower/confd/releases/download/v${CONFDVERSION}/confd-${CONFDVERSION}-linux-amd64 > /usr/local/bin/confd \
|
||||
&& chmod +x /usr/local/bin/confd
|
||||
|
||||
ADD patronictl.py patroni.py docker/entrypoint.sh /
|
||||
ADD patroni /patroni/
|
||||
ADD extras/confd /etc/confd
|
||||
RUN ln -s /patronictl.py /usr/local/bin/patronictl
|
||||
ENV ETCDVERSION 2.0.13
|
||||
RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C /bin --strip=1 --wildcards --no-anchored etcd etcdctl
|
||||
|
||||
### Setting up a simple script that will serve as an entrypoint
|
||||
RUN mkdir /data/ && touch /pgpass /patroni.yml \
|
||||
&& chown postgres:postgres -R /patroni/ /data/ /pgpass /patroni.yml /etc/haproxy /var/run/ /var/lib/ /var/log/ \
|
||||
&& echo 1 > /etc/zookeeper/conf/myid
|
||||
RUN mkdir /data/ && touch /var/log/etcd.log /var/log/etcd.err /pgpass /patroni/postgres.yml
|
||||
RUN chown postgres:postgres -R /patroni/ /data/ /pgpass /var/log/etcd.* /patroni/postgres.yml
|
||||
ADD docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
EXPOSE 2379 5432 8008
|
||||
EXPOSE 4001 5432 2380
|
||||
|
||||
ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
|
||||
USER postgres
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
Alexander Kukushkin <[email protected]>
|
||||
Feike Steenbergen <[email protected]>
|
||||
Oleksii Kliukin <[email protected]>
|
||||
+176
-74
@@ -1,68 +1,34 @@
|
||||
|Build Status| |Coverage Status|
|
||||
|
||||
Patroni: A Template for PostgreSQL HA with ZooKeeper, etcd or Consul
|
||||
Patroni: A Template for PostgreSQL HA with ZooKeeper or etcd
|
||||
------------------------------------------------------------
|
||||
There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
|
||||
|
||||
Patroni is a template for you to create your own customized, high-availability solution using Python and — for maximum accessibility — a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter—or anywhere else—will hopefully find it useful.
|
||||
Patroni was previously known as Governor.
|
||||
|
||||
We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely.
|
||||
*There are many ways to run high availability with PostgreSQL. Here, we
|
||||
present a template for you to create your own customized, high-availability
|
||||
solution using Python and — for maximum accessibility — a distributed
|
||||
configuration store like ZooKeeper or etcd.*
|
||||
|
||||
**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. Please contact us via our Issues Tracker if this describes your team's current setup, and we'll follow up.
|
||||
|
||||
.. contents::
|
||||
:local:
|
||||
:depth: 1
|
||||
:backlinks: none
|
||||
|
||||
==============
|
||||
How Patroni Works
|
||||
==============
|
||||
|
||||
Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features.
|
||||
|
||||
For an example of a Docker-based deployment with Patroni, see `Spilo <https://github.com/zalando/spilo>`__, currently in use at Zalando.
|
||||
|
||||
For additional background info, see:
|
||||
|
||||
* `PostgreSQL HA with Kubernetes and Patroni <https://www.youtube.com/watch?v=iruaCgeG7qs>`__, talk by Josh Berkus at KubeCon 2016 (video)
|
||||
* `Feb. 2016 Zalando Tech blog post <https://tech.zalando.de/blog/zalandos-patroni-a-template-for-high-availability-postgresql/>`__
|
||||
|
||||
================
|
||||
Development Status
|
||||
================
|
||||
|
||||
Patroni is in active development and accepts contributions. See our `Contributing <https://github.com/zalando/patroni/blob/master/README.rst#contributing>`__ section below for more details.
|
||||
|
||||
===========================
|
||||
Technical Requirements/Installation
|
||||
===========================
|
||||
|
||||
**For Mac**
|
||||
|
||||
To install requirements on a Mac, run the following:
|
||||
|
||||
::
|
||||
|
||||
brew install postgresql etcd haproxy libyaml python
|
||||
pip install psycopg2 pyyaml
|
||||
|
||||
===================
|
||||
Running and Configuring
|
||||
===================
|
||||
Getting Started
|
||||
---------------
|
||||
|
||||
To get started, do the following from different terminals:
|
||||
|
||||
::
|
||||
|
||||
> etcd --data-dir=data/etcd
|
||||
> ./patroni.py postgres0.yml
|
||||
> ./patroni.py postgres1.yml
|
||||
|
||||
You will then see a high-availability cluster start up. Test different settings in the YAML files to see how the cluster’s behavior changes. Kill some of the components to see how the system behaves.
|
||||
From there, you will see a high-availability cluster start up. Test
|
||||
different settings in the YAML files to see how its behavior changes. Kill
|
||||
some of the components to see how the system behaves.
|
||||
|
||||
Add more ``postgres*.yml`` files to create an even larger cluster.
|
||||
|
||||
Patroni provides an `HAProxy <http://www.haproxy.org/>`__ configuration, which will give your application a single endpoint for connecting to the cluster's leader. To configure,
|
||||
We provide a haproxy configuration, which will give your application a
|
||||
single endpoint for connecting to the cluster's leader. To configure,
|
||||
run:
|
||||
|
||||
::
|
||||
@@ -73,54 +39,190 @@ run:
|
||||
|
||||
> psql --host 127.0.0.1 --port 5000 postgres
|
||||
|
||||
===============
|
||||
How Patroni Works
|
||||
-----------------
|
||||
|
||||
For a diagram of the high availability decision loop, review this PDF:
|
||||
`postgres-ha.pdf <https://github.com/zalando/patroni/blob/master/postgres-ha.pdf>`__
|
||||
|
||||
YAML Configuration
|
||||
===============
|
||||
------------------
|
||||
|
||||
Go `here <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
|
||||
For an example file, see ``postgres0.yml``. Regarding settings:
|
||||
|
||||
=========================
|
||||
Environment Configuration
|
||||
=========================
|
||||
- *ttl*: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process.
|
||||
- *loop\_wait*: the number of seconds the loop will sleep
|
||||
|
||||
Go `here <https://github.com/zalando/patroni/blob/master/docs/ENVIRONMENT.rst>`__ for comprehensive information about configuring(overriding) settings via environment variables.
|
||||
- *restapi*:
|
||||
- *listen*: IP address + port that Patroni will listen to, to provide health-check information for haproxy.
|
||||
- *connect\_address*: IP address + port through which restapi is accessible.
|
||||
- *auth*: (optional) 'username:password' to protect dangerous REST API endpoints.
|
||||
- *certfile*: (optional) Specifies a file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL.
|
||||
- *keyfile*: (optional) Specifies a file with the secret key in the PEM format.
|
||||
|
||||
- *etcd*:
|
||||
- *scope*: the relative path used on etcd's HTTP API for this deployment; makes it possible to run multiple HA deployments from a single etcd.
|
||||
- *ttl*: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process.
|
||||
- *host*: the host:port for the etcd endpoint.
|
||||
|
||||
- *zookeeper*:
|
||||
- *scope*: the relative path used on etcd's HTTP API for this deployment; makes it possible to run multiple HA deployments from a single etcd.
|
||||
- *session\_timeout*: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process.
|
||||
- *reconnect\_timeout*: how long we should try to reconnect to ZooKeeper after a connection loss. After this timeout, assume that you no longer have a lock and restart in read-only mode.
|
||||
- *hosts*: list of ZooKeeper cluster members in format: ['host1:port1', 'host2:port2', 'etc...']
|
||||
- *exhibitor*: if you are running a ZooKeeper cluster under the Exhibitor supervisory, the following section might interest you:
|
||||
- *poll\_interval*: how often the list of ZooKeeper and Exhibitor nodes should be updated from Exhibitor
|
||||
- *port*: Exhibitor port.
|
||||
- *hosts*: initial list of Exhibitor (ZooKeeper) nodes in format: ['host1', 'host2', 'etc...' ]. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.
|
||||
|
||||
- *bdr*:
|
||||
- *enable*: on if you want to enable BDR
|
||||
- *database*: database name to support BDR (only a single database is supported)
|
||||
|
||||
- *postgresql*:
|
||||
- *name*: the name of the Postgres host. Must be unique for the cluster.
|
||||
- *listen*: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication.
|
||||
- *connect\_address*: IP address + port through which Postgres is accessible from other nodes and applications.
|
||||
- *data\_dir*: file path to initialize and store Postgres data files.
|
||||
- *maximum\_lag\_on\_failover*: the maximum bytes a follower may lag.
|
||||
- *use\_slots*: whether or not to use replication_slots. Must be False for PostgreSQL 9.3. You should comment out max_replication_slots before it becomes ineligible for leader status.
|
||||
|
||||
- *initdb*: List options to be passed on to initdb
|
||||
- *encoding*: default encoding for new databases
|
||||
- *locale*: default locale for new databases
|
||||
- *data-checksums* # When pg_rewind is needed on 9.3, this needs to be enabled
|
||||
|
||||
- *pg\_hba*: list of lines which should be added to pg\_hba.conf.
|
||||
- *- host all all 0.0.0.0/0 md5*.
|
||||
|
||||
- *replication*:
|
||||
- *username*: replication username; user will be created during initialization.
|
||||
- *password*: replication password; user will be created during initialization.
|
||||
- *network*: network setting for replication in pg\_hba.conf.
|
||||
|
||||
- *callbacks* callback scripts to run on certain actions. Patroni will pass the action, role and cluster name. See scripts/aws.py as an example on how to write them.
|
||||
- *on\_start*: a script to run when the cluster starts.
|
||||
- *on\_stop*: a script to run when the cluster stops.
|
||||
- *on\_restart*: a script to run when the cluster restarts.
|
||||
- *on\_reload*: a script to run when configuration reload is triggered.
|
||||
- *on\_role\_change*: a script to run when the cluster is being promoted or demoted.
|
||||
|
||||
- *superuser*:
|
||||
- *password*: password for the Postgres user, set during initialization.
|
||||
|
||||
- *admin*:
|
||||
- *username*: admin username; user is created during initialization. It will have CREATEDB and CREATEROLE privileges.
|
||||
- *password*: admin password; user is created during initialization.
|
||||
|
||||
- *recovery\_conf*: additional configuration settings written to recovery.conf when configuring follower.
|
||||
- *parameters*: list of configuration settings for Postgres. Many of these are required for replication to work.
|
||||
|
||||
- *create\_replica\_methods*: an ordered list of the create methods for turning a patroni node into a new replica.
|
||||
"basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured
|
||||
as its own config item.
|
||||
|
||||
- *replica\_method* for each create_replica_method other than basebackup, you would add a configuration section
|
||||
of the same name. At a minimum, this should include "command" with a full path to the actual script to be
|
||||
executed. Other configuration parameters will be passed along to the script in the form "parameter=value".
|
||||
|
||||
===============
|
||||
Replication Choices
|
||||
===============
|
||||
-------------------
|
||||
|
||||
Patroni uses Postgres' streaming replication, which is asynchronous by default. For more information, see the `Postgres documentation on streaming replication <http://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION>`__.
|
||||
Patroni uses Postgres' streaming replication. By default, this
|
||||
replication is asynchronous. For more information, see the `Postgres
|
||||
documentation on streaming
|
||||
replication <http://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION>`__.
|
||||
|
||||
Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the follower. This setting should be increased or decreased based on business requirements.
|
||||
Patroni's asynchronous replication configuration allows for
|
||||
``maximum_lag_on_failover`` settings. This setting ensures failover will
|
||||
not occur if a follower is more than a certain number of bytes behind
|
||||
the follower. This setting should be increased or decreased based on
|
||||
business requirements.
|
||||
|
||||
When asynchronous replication is not optimal for your use case, investigate Postgres's `synchronous replication <http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION>`__. Synchronous replication ensures consistency across a cluster by confirming that writes are written to a secondary before returning to the connecting client with a success. The cost of synchronous replication: reduced throughput on writes. This throughput will be entirely based on network performance.
|
||||
When asynchronous replication is not optimal for your use case, investigate
|
||||
how Postgres's `synchronous
|
||||
replication <http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION>`__
|
||||
works. Synchronous replication ensures consistency across a cluster by
|
||||
confirming that writes are written to a secondary before returning to
|
||||
the connecting client with a success. The cost of synchronous
|
||||
replication: reduced throughput on writes. This throughput will
|
||||
be entirely based on network performance. In hosted datacenter
|
||||
environments (like AWS, Rackspace, or any network you do not control),
|
||||
synchrous replication significantly increases the variability of write
|
||||
performance. If followers become inaccessible from the leader, the
|
||||
leader effectively becomes readonly.
|
||||
|
||||
In hosted datacenter environments (like AWS, Rackspace, or any network you do not control), synchronous replication significantly increases the variability of write performance. If followers become inaccessible from the leader, the leader effectively becomes read-only.
|
||||
|
||||
To enable a simple synchronous replication test, add the follow lines to the ``parameters`` section of your YAML configuration files:
|
||||
To enable a simple synchronous replication test, add the follow lines to
|
||||
the ``parameters`` section of your YAML configuration files:
|
||||
|
||||
.. code:: YAML
|
||||
|
||||
synchronous_commit: "on"
|
||||
synchronous_standby_names: "*"
|
||||
|
||||
When using synchronous replication, use at least three Postgres data nodes to ensure write availability if one host fails.
|
||||
When using synchronous replication, use at least three Postgres data nodes
|
||||
to ensure write availability if one host fails.
|
||||
|
||||
Choosing your replication schema is dependent on your business considerations. Investigate both async and sync replication, as well as other HA solutions, to determine which solution is best for you.
|
||||
Choosing your replication schema is dependent on your business
|
||||
considerations. Investigate both async and sync replication, as well as other
|
||||
HA solutions, to determine which solution is best for you.
|
||||
|
||||
You can also use BDR (bi-directional replication) if you have a compatible
|
||||
PostgreSQL version with the BDR plugin installed (see http://bdr-project.org/docs/next/installation.html).
|
||||
It will require adding a BDR shared_library to your configuration, as well as
|
||||
setting the following options for postgresql (see http://bdr-project.org/docs/next/settings-prerequisite.html):
|
||||
|
||||
.. code:: YAML
|
||||
|
||||
max_worker_processes: 10
|
||||
max_replication_slots: 10
|
||||
max_wal_senders: 10
|
||||
shared_preload_libraries: 'bdr'
|
||||
track_commit_timestamp: 'on'
|
||||
wal_level: 'logical'
|
||||
|
||||
At the moment Patroni BDR is not compatible with a streaming replication,
|
||||
if BDR is enabled normal replica node won't be able to join. This is not
|
||||
a principal limitation of BDR, and we might resolve this in the future
|
||||
(although 'promotion' will only work between nodes running a physical
|
||||
replication, i.e. a replica won't be able to attach to a different
|
||||
multimaster node).
|
||||
|
||||
Another limitation is that only one database is supported at the moment.
|
||||
BDR requires the replication user to be also a superuser, so you might
|
||||
want to excersie extra caution when choosing the password for this user.
|
||||
|
||||
===============================
|
||||
Applications Should Not Use Superusers
|
||||
===============================
|
||||
--------------------------------------
|
||||
|
||||
When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
|
||||
When connecting from an application, always use a non-superuser. Patroni
|
||||
requires access to the database to function properly. By using a
|
||||
superuser from an application, you can potentially use the entire
|
||||
connection pool, including the connections reserved for superusers with
|
||||
the ``superuser_reserved_connections`` setting. If Patroni cannot access
|
||||
the Primary because the connection pool is full, behavior will be
|
||||
undesireable.
|
||||
|
||||
================
|
||||
Contributing
|
||||
================
|
||||
Patroni accepts contributions from the open-source community; see the `Issues Tracker <https://github.com/zalando/patroni/issues>`__ for current needs.
|
||||
Requirements on a Mac
|
||||
---------------------
|
||||
|
||||
Before making a contribution, please let us know by posting a comment to the relevant issue.
|
||||
If you would like to propose a new feature, please first file a new issue explaining the feature you’d like to create.
|
||||
Run the following on a Mac to install requirements:
|
||||
|
||||
::
|
||||
|
||||
brew install postgresql etcd haproxy libyaml python
|
||||
pip install psycopg2 pyyaml
|
||||
|
||||
Notice
|
||||
------
|
||||
|
||||
There are many different ways to do HA with PostgreSQL: See `the
|
||||
PostgreSQL
|
||||
documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__
|
||||
for a complete list.
|
||||
|
||||
We call Patroni a "template" because it is far from being a one-size-fits-all
|
||||
or plug-and-play replication system. It will have its own caveats. Use wisely.
|
||||
|
||||
.. |Build Status| image:: https://travis-ci.org/zalando/patroni.svg?branch=master
|
||||
:target: https://travis-ci.org/zalando/patroni
|
||||
|
||||
@@ -1,12 +1,4 @@
|
||||
Failover
|
||||
========
|
||||
- When determining who should become master, include the minor version of PostgreSQL in the decision.
|
||||
|
||||
Configuration
|
||||
==============
|
||||
- Provide a way to change pg_hba.conf of a running cluster on the Patroni level, without changing individual nodes.
|
||||
- Provide hooks to store and retrieve cluster-wide passwords without exposing them in a plain-text form to unauthorized users.
|
||||
|
||||
Documentation
|
||||
==============
|
||||
- Document how to run cascading replication and possibly initialize the cluster without an access to the master node.
|
||||
- When determining who should become master, include the minor version of PostgreSQL in the decision
|
||||
- Create a way to disable governance of a cluster, something like the existence of a "nogover" or "admin" file in PGDATA will stop governor from changing the cluster state
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
# docker compose file for running a 3-node PostgreSQL cluster
|
||||
# with etcd as the SIS
|
||||
|
||||
patroni_etcd:
|
||||
container_name: patroni_etcd
|
||||
image: patroni
|
||||
command: --etcd
|
||||
|
||||
dbnode1:
|
||||
image: patroni
|
||||
hostname: dbnode1
|
||||
links:
|
||||
- patroni_etcd:patroni_etcd
|
||||
volumes:
|
||||
- ./patroni:/patroni
|
||||
env_file: docker/patroni-secrets.env
|
||||
environment:
|
||||
PATRONI_ETCD_HOST: patroni_etcd:2379
|
||||
PATRONI_NAME: dbnode1
|
||||
PATRONI_SCOPE: testcluster
|
||||
|
||||
dbnode2:
|
||||
image: patroni
|
||||
hostname: dbnode2
|
||||
links:
|
||||
- patroni_etcd:patroni_etcd
|
||||
volumes:
|
||||
- ./patroni:/patroni
|
||||
env_file: docker/patroni-secrets.env
|
||||
environment:
|
||||
PATRONI_ETCD_HOST: patroni_etcd:2379
|
||||
PATRONI_NAME: dbnode2
|
||||
PATRONI_SCOPE: testcluster
|
||||
|
||||
dbnode3:
|
||||
image: patroni
|
||||
hostname: dbnode3
|
||||
links:
|
||||
- patroni_etcd:patroni_etcd
|
||||
volumes:
|
||||
- ./patroni:/patroni
|
||||
env_file: docker/patroni-secrets.env
|
||||
environment:
|
||||
PATRONI_ETCD_HOST: patroni_etcd:2379
|
||||
PATRONI_NAME: dbnode3
|
||||
PATRONI_SCOPE: testcluster
|
||||
|
||||
haproxy:
|
||||
image: patroni
|
||||
links:
|
||||
- patroni_etcd:patroni_etcd
|
||||
ports:
|
||||
- "5000"
|
||||
- "5001"
|
||||
environment:
|
||||
PATRONI_ETCD_HOST: patroni_etcd:2379
|
||||
PATRONI_SCOPE: testcluster
|
||||
command: --confd
|
||||
@@ -67,38 +67,24 @@ while getopts "$optspec" optchar; do
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z ${PATRONI_SCOPE} ]; then
|
||||
PATRONI_SCOPE=$(cat /dev/urandom | LC_ALL=C tr -dc 'a-z0-9' | head -c 8)
|
||||
fi
|
||||
|
||||
function docker_run()
|
||||
function random_name()
|
||||
{
|
||||
local name=$1
|
||||
shift
|
||||
container=$(docker run -d --name=$name $*)
|
||||
container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${container})
|
||||
echo "Started container ${name}, ip=${container_ip}"
|
||||
cat /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | head -c 8
|
||||
}
|
||||
|
||||
if [ -z ${PATRONI_SCOPE} ]
|
||||
then
|
||||
PATRONI_SCOPE=$(random_name)
|
||||
fi
|
||||
|
||||
ETCD_CONTAINER="${PATRONI_SCOPE}_etcd"
|
||||
docker_run ${ETCD_CONTAINER} ${DOCKER_IMAGE} --etcd
|
||||
etcd_container=$(docker run -P -d --name="${PATRONI_SCOPE}_etcd" "${DOCKER_IMAGE}" --etcd-only)
|
||||
etcd_container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${etcd_container})
|
||||
echo "The etcd container is ${etcd_container}, ip=${etcd_container_ip}"
|
||||
|
||||
DOCKER_ARGS="--link=${ETCD_CONTAINER}:${ETCD_CONTAINER} -e PATRONI_SCOPE=${PATRONI_SCOPE} -e PATRONI_ETCD_HOST=${ETCD_CONTAINER}:2379"
|
||||
PATRONI_ENV=$(sed 's/#.*//g' docker/patroni-secrets.env | sed -n 's/^PATRONI_.*$/-e &/p' | tr '\n' ' ')
|
||||
PATRONI_VOLUME="-v $(dirname $(dirname $(realpath $0)))/patroni:/patroni"
|
||||
|
||||
for i in $(seq 1 "${MEMBERS}"); do
|
||||
container_name=postgres${i}
|
||||
docker_run "${PATRONI_SCOPE}_${container_name}" \
|
||||
$PATRONI_VOLUME \
|
||||
$DOCKER_ARGS \
|
||||
$PATRONI_ENV \
|
||||
-e PATRONI_NAME=${container_name} \
|
||||
${DOCKER_IMAGE}
|
||||
for i in $(seq 1 "${MEMBERS}")
|
||||
do
|
||||
container_name=$(random_name)
|
||||
patroni_container=$(docker run -P -d --name="${PATRONI_SCOPE}_${container_name}" "${DOCKER_IMAGE}" --etcd="${etcd_container_ip}:4001" --name="${PATRONI_SCOPE}")
|
||||
patroni_container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${patroni_container})
|
||||
echo "Started Patroni container ${patroni_container}, ip=${patroni_container_ip}"
|
||||
done
|
||||
|
||||
docker_run "${PATRONI_SCOPE}_haproxy" \
|
||||
-p=5000 -p=5001 \
|
||||
$DOCKER_ARGS \
|
||||
${DOCKER_IMAGE} --confd
|
||||
|
||||
+88
-58
@@ -7,52 +7,49 @@ Usage: $0
|
||||
|
||||
Options:
|
||||
|
||||
--etcd Do not run Patroni, run a standalone etcd
|
||||
--confd Do not run Patroni, run a standalone confd
|
||||
--zookeeper Do not run Patroni, run a standalone zookeeper
|
||||
--etcd ETCD Provide an external etcd to connect to
|
||||
--name NAME Give the cluster a specific name
|
||||
--etcd-only Do not run Patroni, run a standalone etcd
|
||||
|
||||
Examples:
|
||||
|
||||
$0 --etcd
|
||||
$0 --confd
|
||||
$0 --zookeeper
|
||||
$0 --etcd=127.17.0.84:4001
|
||||
$0 --etcd-only
|
||||
$0
|
||||
$0 --name=true_scotsman
|
||||
__EOF__
|
||||
}
|
||||
|
||||
DOCKER_IP=$(hostname --ip-address)
|
||||
PATRONI_SCOPE=${PATRONI_SCOPE:-batman}
|
||||
ETCD_ARGS="--data-dir /tmp/etcd.data -advertise-client-urls=http://${DOCKER_IP}:2379 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://0.0.0.0:2380"
|
||||
|
||||
optspec=":vh-:"
|
||||
while getopts "$optspec" optchar; do
|
||||
case "${optchar}" in
|
||||
-)
|
||||
case "${OPTARG}" in
|
||||
confd)
|
||||
haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D
|
||||
CONFD="confd -prefix=${PATRONI_NAMESPACE:-/service}/$PATRONI_SCOPE -interval=10 -backend"
|
||||
if [ ! -z ${PATRONI_ZOOKEEPER_HOSTS} ]; then
|
||||
while ! /usr/share/zookeeper/bin/zkCli.sh -server ${PATRONI_ZOOKEEPER_HOSTS} ls /; do
|
||||
sleep 1
|
||||
done
|
||||
exec $CONFD zookeeper -node ${PATRONI_ZOOKEEPER_HOSTS}
|
||||
else
|
||||
while ! curl -s ${PATRONI_ETCD_HOST}/v2/members | jq -r '.members[0].clientURLs[0]' | grep -q http; do
|
||||
sleep 1
|
||||
done
|
||||
exec $CONFD etcd -node $PATRONI_ETCD_HOST
|
||||
fi
|
||||
;;
|
||||
etcd)
|
||||
exec etcd $ETCD_ARGS
|
||||
;;
|
||||
zookeeper)
|
||||
exec /usr/share/zookeeper/bin/zkServer.sh start-foreground
|
||||
etcd-only)
|
||||
exec etcd --data-dir /tmp/etcd.data \
|
||||
-advertise-client-urls=http://${DOCKER_IP}:4001 \
|
||||
-listen-client-urls=http://0.0.0.0:4001 \
|
||||
-listen-peer-urls=http://0.0.0.0:2380
|
||||
exit 0
|
||||
;;
|
||||
cheat)
|
||||
CHEAT=1
|
||||
;;
|
||||
name)
|
||||
PATRONI_SCOPE="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
|
||||
;;
|
||||
name=*)
|
||||
PATRONI_SCOPE=${OPTARG#*=}
|
||||
;;
|
||||
etcd)
|
||||
ETCD_CLUSTER="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
|
||||
;;
|
||||
etcd=*)
|
||||
ETCD_CLUSTER=${OPTARG#*=}
|
||||
;;
|
||||
help)
|
||||
usage
|
||||
exit 0
|
||||
@@ -73,43 +70,76 @@ while getopts "$optspec" optchar; do
|
||||
esac
|
||||
done
|
||||
|
||||
## We start an etcd
|
||||
if [[ -z ${PATRONI_ETCD_HOST} && -z ${PATRONI_ZOOKEEPER_HOSTS} ]]; then
|
||||
etcd $ETCD_ARGS > /var/log/etcd.log 2> /var/log/etcd.err &
|
||||
export PATRONI_ETCD_HOST="127.0.0.1:2379"
|
||||
if [ -z ${ETCD_CLUSTER} ]
|
||||
then
|
||||
etcd --data-dir /tmp/etcd.data \
|
||||
-advertise-client-urls=http://${DOCKER_IP}:4001 \
|
||||
-listen-client-urls=http://0.0.0.0:4001 \
|
||||
-listen-peer-urls=http://0.0.0.0:2380 > /var/log/etcd.log 2> /var/log/etcd.err &
|
||||
ETCD_CLUSTER="127.0.0.1:4001"
|
||||
fi
|
||||
|
||||
export PATRONI_SCOPE
|
||||
export PATRONI_NAME="${PATRONI_NAME:-${HOSTNAME}}"
|
||||
export PATRONI_RESTAPI_CONNECT_ADDRESS="${DOCKER_IP}:8008"
|
||||
export PATRONI_RESTAPI_LISTEN="0.0.0.0:8008"
|
||||
export PATRONI_admin_PASSWORD="${PATRONI_admin_PASSWORD:=admin}"
|
||||
export PATRONI_admin_OPTIONS="${PATRONI_admin_OPTIONS:-createdb, createrole}"
|
||||
export PATRONI_POSTGRESQL_CONNECT_ADDRESS="${DOCKER_IP}:5432"
|
||||
export PATRONI_POSTGRESQL_LISTEN="0.0.0.0:5432"
|
||||
export PATRONI_POSTGRESQL_DATA_DIR="data/${PATRONI_SCOPE}"
|
||||
export PATRONI_REPLICATION_USERNAME="${PATRONI_REPLICATION_USERNAME:-replicator}"
|
||||
export PATRONI_REPLICATION_PASSWORD="${PATRONI_REPLICATION_PASSWORD:-abcd}"
|
||||
export PATRONI_SUPERUSER_USERNAME="${PATRONI_SUPERUSER_USERNAME:-postgres}"
|
||||
export PATRONI_SUPERUSER_PASSWORD="${PATRONI_SUPERUSER_PASSWORD:-postgres}"
|
||||
export PATRONI_POSTGRESQL_PGPASS="$HOME/.pgpass"
|
||||
|
||||
cat > /patroni.yml <<__EOF__
|
||||
bootstrap:
|
||||
dcs:
|
||||
postgresql:
|
||||
use_pg_rewind: true
|
||||
cat > /patroni/postgres.yml <<__EOF__
|
||||
|
||||
ttl: &ttl 30
|
||||
loop_wait: &loop_wait 10
|
||||
scope: &scope '${PATRONI_SCOPE}'
|
||||
namespace: 'patroni'
|
||||
restapi:
|
||||
listen: 0.0.0.0:8008
|
||||
connect_address: ${DOCKER_IP}:8008
|
||||
etcd:
|
||||
scope: *scope
|
||||
ttl: *ttl
|
||||
host: ${ETCD_CLUSTER}
|
||||
bdr:
|
||||
enable: 'on'
|
||||
database: 'bdrtest'
|
||||
postgresql:
|
||||
name: ${HOSTNAME}
|
||||
scope: *scope
|
||||
listen: 0.0.0.0:5432
|
||||
connect_address: ${DOCKER_IP}:5432
|
||||
data_dir: data/postgresql0
|
||||
maximum_lag_on_failover: 1048576 # 1 megabyte in bytes
|
||||
pg_hba:
|
||||
- host all all 0.0.0.0/0 md5
|
||||
- hostssl all all 0.0.0.0/0 md5
|
||||
- host replication replicator ${DOCKER_IP}/16 md5
|
||||
replication:
|
||||
username: replicator
|
||||
password: rep-pass
|
||||
network: 127.0.0.1/32
|
||||
superuser:
|
||||
password: zalando
|
||||
restore: patroni/scripts/restore.py
|
||||
admin:
|
||||
username: admin
|
||||
password: admin
|
||||
parameters:
|
||||
archive_mode: "on"
|
||||
wal_level: hot_standby
|
||||
archive_command: 'true'
|
||||
max_wal_senders: 20
|
||||
listen_addresses: 0.0.0.0
|
||||
checkpoint_segments: 64
|
||||
wal_keep_segments: 64
|
||||
archive_timeout: 1800s
|
||||
max_replication_slots: 20
|
||||
hot_standby: "on"
|
||||
max_worker_processes: 10
|
||||
max_replication_slots: 10
|
||||
max_wal_senders: 10
|
||||
shared_preload_libraries: 'bdr'
|
||||
track_commit_timestamp: true
|
||||
wal_level: 'logical'
|
||||
__EOF__
|
||||
|
||||
mkdir -p "$HOME/.config/patroni"
|
||||
[ -h "$HOME/.config/patroni/patronictl.yaml" ] || ln -s /patroni.yml "$HOME/.config/patroni/patronictl.yaml"
|
||||
cat /patroni/postgres.yml
|
||||
|
||||
[ -z $CHEAT ] && exec python /patroni.py /patroni.yml
|
||||
|
||||
while true; do
|
||||
sleep 60
|
||||
done
|
||||
if [ ! -z $CHEAT ]
|
||||
then
|
||||
exec bash
|
||||
else
|
||||
exec python /patroni.py /patroni/postgres.yml
|
||||
fi
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
PATRONI_RESTAPI_USERNAME=admin
|
||||
PATRONI_RESTAPI_PASSWORD=admin
|
||||
PATRONI_SUPERUSER_USERNAME=postgres
|
||||
PATRONI_SUPERUSER_PASSWORD=postgres
|
||||
PATRONI_REPLICATION_USERNAME=replicator
|
||||
PATRONI_REPLICATION_PASSWORD=replicate
|
||||
PATRONI_admin_PASSWORD=admin
|
||||
PATRONI_admin_OPTIONS=createdb,createrole
|
||||
@@ -1,58 +0,0 @@
|
||||
==================================
|
||||
Environment Configuration Settings
|
||||
==================================
|
||||
|
||||
It is possible to override some of the configuration parameters defined in the Patroni configuration file using the system environment variables. This document lists all environment variables handled by Patroni. The values set via those variables always take precedence over the ones set in the Patroni configuration file.
|
||||
|
||||
Global/Universal
|
||||
----------------
|
||||
- **PATRONI\_CONFIGURATION**: it is possible to set the entire configuration for the Patroni via ``PATRONI_CONFIGURATION`` environment variable. In this case any other environment variables will not be considered!
|
||||
- **PATRONI\_NAME**: name of the node where the current instance of Patroni is running. Must be unique for the cluster.
|
||||
- **PATRONI\_NAMESPACE**: path within the configuration store where Patroni will keep information about the cluster. Default value: "/service"
|
||||
- **PATRONI\_SCOPE**: cluster name
|
||||
|
||||
Bootstrap configuration
|
||||
-----------------------
|
||||
It is possible to create new database users right after the successful initialization of a new cluster. This process is defined by the following variables:
|
||||
|
||||
- **PATRONI\_<username>\_PASSWORD='<password>'**
|
||||
- **PATRONI\_<username>\_OPTIONS='list,of,options'**
|
||||
|
||||
Example: defining ``PATRONI_admin_PASSWORD=strongpasswd`` and ``PATRONI_admin_OPTIONS='createrole,createdb'`` will cause creation of the user **admin** with the password **strongpasswd** that is allowed to create other users and databases.
|
||||
|
||||
Consul
|
||||
------
|
||||
- **PATRONI\_CONSUL\_HOST**: the host:port for the Consul endpoint.
|
||||
|
||||
Etcd
|
||||
----
|
||||
- **PATRONI\_ETCD\_HOST**: the host:port for the etcd endpoint.
|
||||
|
||||
Exhibitor
|
||||
---------
|
||||
- **PATRONI\_EXHIBITOR\_HOSTS**: initial list of Exhibitor (ZooKeeper) nodes in format: 'host1,host2,etc...'. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.
|
||||
- **PATRONI\_EXHIBITOR\_PORT**: Exhibitor port.
|
||||
|
||||
PostgreSQL
|
||||
----------
|
||||
- **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
|
||||
- **PATRONI\_POSTGRESQL\_CONNECT\_ADDRESS**: IP address + port through which Postgres is accessible from other nodes and applications.
|
||||
- **PATRONI\_POSTGRESQL\_DATA\_DIR**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
|
||||
- **PATRONI\_POSTGRESQL\_PGPASS**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni.
|
||||
- **PATRONI\_REPLICATION\_USERNAME**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication
|
||||
- **PATRONI\_REPLICATION\_PASSWORD**: replication password; the user will be created during initialization.
|
||||
- **PATRONI\_SUPERUSER\_USERNAME**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres. Also this user is used by pg_rewind.
|
||||
- **PATRONI\_SUPERUSER\_PASSWORD**: password for the superuser, set during initialization (initdb).
|
||||
|
||||
REST API
|
||||
--------
|
||||
- **PATRONI\_RESTAPI\_CONNECT\_ADDRESS**: IP address and port to access the REST API.
|
||||
- **PATRONI\_RESTAPI\_LISTEN**: IP address and port that Patroni will listen to, to provide health-check information for HAProxy.
|
||||
- **PATRONI\_RESTAPI\_USERNAME**: Basic-auth username to protect unsafe REST API endpoints.
|
||||
- **PATRONI\_RESTAPI\_PASSWORD**: Basic-auth password to protect unsafe REST API endpoints.
|
||||
- **PATRONI\_RESTAPI\_CERTFILE**: Specifies the file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL.
|
||||
- **PATRONI\_RESTAPI\_KEYFILE**: Specifies the file with the secret key in the PEM format.
|
||||
|
||||
ZooKeeper
|
||||
---------
|
||||
- **PATRONI\_ZOOKEEPER\_HOSTS**: comma separated list of ZooKeeper cluster members: "'host1:port1','host2:port2','etc...'". It is important to quote every single entity!
|
||||
@@ -1,91 +0,0 @@
|
||||
===========================
|
||||
YAML Configuration Settings
|
||||
===========================
|
||||
|
||||
Global/Universal
|
||||
----------------
|
||||
- **name**: the name of the host. Must be unique for the cluster.
|
||||
- **namespace**: path within the configuration store where Patroni will keep information about the cluster. Default value: "/service"
|
||||
- **scope**: cluster name
|
||||
|
||||
Bootstrap configuration
|
||||
-----------------------
|
||||
- **dcs**: This section will be written into `/<namespace>/<scope>/config` of a given configuration store after initializing of new cluster. This is the global configuration for the cluster. If you want to change some parameters for all cluster nodes - just do it in DCS (or via Patroni API) and all nodes will apply this configuration.
|
||||
- **loop\_wait**: the number of seconds the loop will sleep. Default value: 10
|
||||
- **ttl**: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process. Default value: 30
|
||||
- **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election.
|
||||
- **postgresql**:
|
||||
- **use\_pg\_rewind**:whether or not to use pg_rewind
|
||||
- **use\_slots**: whether or not to use replication_slots. Must be False for PostgreSQL 9.3. You should comment out max_replication_slots before it becomes ineligible for leader status.
|
||||
- **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower.
|
||||
- **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.
|
||||
- **initdb**: List options to be passed on to initdb.
|
||||
- **- data-checksums**: Must be enabled when pg_rewind is needed on 9.3.
|
||||
- **- encoding: UTF8**: default encoding for new databases.
|
||||
- **- locale: UTF8**: default locale for new databases.
|
||||
- **pg\_hba**: list of lines that you should add to pg\_hba.conf.
|
||||
- **- host all all 0.0.0.0/0 md5**.
|
||||
- **- host replication replicator 127.0.0.1/32 md5**: A line like this is required for replication.
|
||||
- **users**: Some additional users users which needs to be created after initializing new cluster
|
||||
- **admin**: the name of user
|
||||
- **password: zalando**:
|
||||
- **options**: list of options for CREATE USER statement
|
||||
- **- createrole**
|
||||
- **- createdb**
|
||||
|
||||
Consul
|
||||
------
|
||||
- **host**: the host:port for the Consul endpoint.
|
||||
|
||||
Etcd
|
||||
----
|
||||
- **host**: the host:port for the etcd endpoint.
|
||||
|
||||
Exhibitor
|
||||
---------
|
||||
- **hosts**: initial list of Exhibitor (ZooKeeper) nodes in format: 'host1,host2,etc...'. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.
|
||||
- **poll\_interval**: how often the list of ZooKeeper and Exhibitor nodes should be updated from Exhibitor
|
||||
- **port**: Exhibitor port.
|
||||
|
||||
PostgreSQL
|
||||
----------
|
||||
- **authentication**:
|
||||
- **superuser**:
|
||||
- **username**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres.
|
||||
- **password**: password for the superuser, set during initialization (initdb).
|
||||
- **replication**:
|
||||
- **username**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication
|
||||
- **password**: replication password; the user will be created during initialization.
|
||||
- **callbacks**: callback scripts to run on certain actions. Patroni will pass the action, role and cluster name. (See scripts/aws.py as an example of how to write them.)
|
||||
- **on\_reload**: run this script when configuration reload is triggered.
|
||||
- **on\_restart**: run this script when the cluster restarts.
|
||||
- **on\_role\_change**: run this script when the cluster is being promoted or demoted.
|
||||
- **on\_start**: run this script when the cluster starts.
|
||||
- **on\_stop**: run this script when the cluster stops.
|
||||
- **connect\_address**: IP address + port through which Postgres is accessible from other nodes and applications.
|
||||
- **create\_replica\_methods**: an ordered list of the create methods for turning a Patroni node into a new replica. "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its own config item.
|
||||
- **data\_dir**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
|
||||
- **listen**: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
|
||||
- **pgpass**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni.
|
||||
- **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower.
|
||||
- **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.
|
||||
- **pg\_ctl\_timeout**: How long should pg_ctl wait when doing ``start``, ``stop`` or ``restart``. Default value is 60 seconds.
|
||||
- **use\_pg\_rewind**: try to use pg\_rewind on the former leader when it joins cluster as a replica.
|
||||
- **remove\_data\_directory\_on\_rewind\_failure**: If this option is enabled, Patroni will remove postgres data directory and recreate replica. Otherwise it will try to follow the new leader. Default value is **false**.
|
||||
- **replica\_method** for each create_replica_method other than basebackup, you would add a configuration section of the same name. At a minimum, this should include "command" with a full path to the actual script to be executed. Other configuration parameters will be passed along to the script in the form "parameter=value".
|
||||
|
||||
REST API
|
||||
--------
|
||||
- **connect\_address**: IP address and port to access the REST API.
|
||||
- **listen**: IP address and port that Patroni will listen to, to provide health-check information for HAProxy.
|
||||
- **Optional**:
|
||||
- **authentication**:
|
||||
- **username**: Basic-auth username to protect unsafe REST API endpoints.
|
||||
- **password**: Basic-auth password to protect unsafe REST API endpoints.
|
||||
|
||||
- **certfile**: Specifies the file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL.
|
||||
- **keyfile**: Specifies the file with the secret key in the PEM format.
|
||||
|
||||
ZooKeeper
|
||||
----------
|
||||
- **hosts**: list of ZooKeeper cluster members in format: ['host1:port1', 'host2:port2', 'etc...'].
|
||||
@@ -1,225 +0,0 @@
|
||||
Patroni configuration
|
||||
=====================
|
||||
|
||||
Patroni configuration is stored in the DCS (Distributed Configuration Store). There are 3 types of configuration:
|
||||
|
||||
- Dynamic configuration.
|
||||
These options can be set in DCS at any time. If the options changed are not part of the startup configuration,
|
||||
they are applied asynchronously (upon the next wake up cycle) to every node, which gets subsequently reloaded.
|
||||
If the node requires a restart to apply the configuration (for options with context postmaster, if their values
|
||||
have changed), a special flag, ``pending_restart`` indicating this, is set in the members.data JSON.
|
||||
Additionally, the node status also indicates this, by showing ``"restart_pending": true``.
|
||||
|
||||
- Local `configuration <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ (patroni.yml).
|
||||
These options are defined in the configuration file and take precedence over dynamic configuration.
|
||||
patroni.yml could be changed and reload in runtime (without restart of Patroni) by sending SIGHUP to the Patroni process or by performing ``POST /reload`` REST-API request.
|
||||
|
||||
- Environment `configuration <https://github.com/zalando/patroni/blob/master/docs/ENVIRONMENT.rst>`__ .
|
||||
It is possible to set/override some of the "Local" configuration parameters with environment variables.
|
||||
Environment configuration is very useful when you are running in a dynamic environment and you don't know some of the parameters in advance (for example it's not possible to know you external IP address when you are running inside ``docker``).
|
||||
|
||||
Some of the PostgreSQL parameters must hold the same values on the master and the replicas. For those, values set either in the local patroni configuration files or via the environment variables take no effect. To alter or set their values one must change the shared configuration in the DCS. Below is the actual list of such parameters together with the default values:
|
||||
|
||||
- max_connections: 100
|
||||
- max_locks_per_transaction: 64
|
||||
- max_worker_processes: 8
|
||||
- max_prepared_transactions: 0
|
||||
- wal_level: hot_standby
|
||||
- wal_log_hints: on
|
||||
- track_commit_timestamp: off
|
||||
|
||||
For the parameters below, PostgreSQL does not require equal values among the master and all the replicas. However, considering the possibility of a replica to become the master at any time, it doesn't really make sense to set them differently; therefore, Patroni restricts setting their values to the Dynamic configuration
|
||||
|
||||
- max_wal_senders: 5
|
||||
- max_replication_slots: 5
|
||||
- wal_keep_segments: 8
|
||||
|
||||
These parameters are validated to ensure they are sane, or meet a minimum value.
|
||||
|
||||
There are some other Postgres parameters controlled by Patroni:
|
||||
|
||||
- listen_addresses - is set either from ``postgresql.listen`` or from ``PATRONI_POSTGRESQL_LISTEN`` environment variable
|
||||
- port - is set either from ``postgresql.listen`` or from ``PATRONI_POSTGRESQL_LISTEN`` environment variable
|
||||
- cluster_name - is set either from ``scope`` or from ``PATRRONI_SCOPE`` environment variable
|
||||
- hot_standby: on
|
||||
|
||||
To be on the safe side parameters from the above lists are not written into ``postgresql.conf``, but passed as a list of arguments to the ``pg_ctl start`` which gives them the highest precedence, even above `ALTER SYSTEM <https://www.postgresql.org/docs/current/static/sql-altersystem.html>`__
|
||||
|
||||
|
||||
When applying the local or dynamic configuration options, the following actions are taken:
|
||||
|
||||
- The node first checks if there is a postgresql.base.conf.
|
||||
- If it exists, it contains the renamed "original" configuration.
|
||||
- If it doesn't, the original postgresql.conf is taken and renamed to postgresql.base.conf.
|
||||
- The dynamic options (with the exceptions above) are dumped into the postgresql.conf and an include is set in
|
||||
postgresql.conf to postgresql.base.conf. Therefore, we would be able to apply new options without re-reading the configuration file to check if the include is present not.
|
||||
- Some parameters that are essential for Patroni to manage the cluster are overridden using the command line.
|
||||
- If some of the options that require restart are changed (we should look at the context in pg_settings and at the actual
|
||||
values of those options), a pending_restart flag of a given node is set. This flag is reset on any restart.
|
||||
|
||||
The parameters would be applied in the following order (run-time are given the highest priority):
|
||||
|
||||
1. load parameters from file `postgresql.base.conf`
|
||||
2. load parameters from file `postgresql.conf`
|
||||
3. load parameters from file `postgresql.auto.conf`
|
||||
4. run-time parameter using `-o --name=value`
|
||||
|
||||
This allows configuration for all the nodes (2), configuration for a specific node using `ALTER SYSTEM` (3) and ensures that parameters essential to the running of Patroni are enforced. (4)
|
||||
|
||||
|
||||
Also, the following Patroni configuration options can be changed only dynamically:
|
||||
|
||||
- ttl: 30
|
||||
- loop_wait: 10
|
||||
- retry_timeouts: 10
|
||||
- maximum_lag_on_failover: 1048576
|
||||
- postgresql.use_slots: true
|
||||
|
||||
Upon changing these options, Patroni will read the relevant section of the configuration stored in DCS and change its
|
||||
run-time values.
|
||||
|
||||
Patroni nodes are dumping the state of the DCS options to disk upon for every change of the configuration into the file ``patroni.dynamic.json`` located in the Postgres data directory. Only the master is allowed to restore these options from the on-disk dump if these are completely absent from the DCS or if they are invalid.
|
||||
|
||||
REST API
|
||||
========
|
||||
|
||||
We provide a REST API endpoint for working with dynamic configuration.
|
||||
|
||||
GET /config
|
||||
-----------
|
||||
Get current version of dynamic configuration.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ curl -s localhost:8008/config | jq .
|
||||
{
|
||||
"ttl": 30,
|
||||
"loop_wait": 10,
|
||||
"retry_timeout": 10,
|
||||
"maximum_lag_on_failover": 1048576,
|
||||
"postgresql": {
|
||||
"use_slots": true,
|
||||
"use_pg_rewind": true,
|
||||
"parameters": {
|
||||
"hot_standby": "on",
|
||||
"wal_log_hints": "on",
|
||||
"wal_keep_segments": 8,
|
||||
"wal_level": "hot_standby",
|
||||
"max_wal_senders": 5,
|
||||
"max_replication_slots": 5,
|
||||
"max_connections": "100"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
PATCH /config
|
||||
-------------
|
||||
Change existing configuration.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ curl -s -XPATCH -d \
|
||||
'{"loop_wait":5,"ttl":20,"postgresql":{"parameters":{"max_connections":"101"}}}' \
|
||||
http://localhost:8008/config | jq .
|
||||
{
|
||||
"ttl": 20,
|
||||
"loop_wait": 5,
|
||||
"maximum_lag_on_failover": 1048576,
|
||||
"retry_timeout": 10,
|
||||
"postgresql": {
|
||||
"use_slots": true,
|
||||
"use_pg_rewind": true,
|
||||
"parameters": {
|
||||
"hot_standby": "on",
|
||||
"wal_log_hints": "on",
|
||||
"wal_keep_segments": 8,
|
||||
"wal_level": "hot_standby",
|
||||
"max_wal_senders": 5,
|
||||
"max_replication_slots": 5,
|
||||
"max_connections": "101"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
The above REST API call patches the existing configuration and returns the new configuration.
|
||||
|
||||
Let's check that the node processed this configuration. First of all it should start printing log lines every 5 seconds (loop_wait=5). The change of "max_connections" requires a restart, so the "restart_pending" flag should be exposed:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ curl -s http://localhost:8008/patroni | jq .
|
||||
{
|
||||
"pending_restart": true,
|
||||
"database_system_identifier": "6287881213849985952",
|
||||
"postmaster_start_time": "2016-06-13 13:13:05.211 CEST",
|
||||
"xlog": {
|
||||
"location": 2197818976
|
||||
},
|
||||
"patroni": {
|
||||
"scope": "batman",
|
||||
"version": "1.0"
|
||||
},
|
||||
"state": "running",
|
||||
"role": "master",
|
||||
"server_version": 90503
|
||||
}
|
||||
|
||||
Removing parameters:
|
||||
|
||||
If you want to remove (reset) some setting just patch it with ``null``:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ curl -s -XPATCH -d \
|
||||
'{"postgresql":{"parameters":{"max_connections":null}}}' \
|
||||
http://localhost:8008/config | jq .
|
||||
{
|
||||
"ttl": 20,
|
||||
"loop_wait": 5,
|
||||
"retry_timeout": 10,
|
||||
"maximum_lag_on_failover": 1048576,
|
||||
"postgresql": {
|
||||
"use_slots": true,
|
||||
"use_pg_rewind": true,
|
||||
"parameters": {
|
||||
"hot_standby": "on",
|
||||
"unix_socket_directories": ".",
|
||||
"wal_keep_segments": 8,
|
||||
"wal_level": "hot_standby",
|
||||
"wal_log_hints": "on",
|
||||
"max_wal_senders": 5,
|
||||
"max_replication_slots": 5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Above call removes ``postgresql.parameters.max_connections`` from the dynamic configuration.
|
||||
|
||||
PUT /config
|
||||
-----------
|
||||
|
||||
It's also possible to perform the full rewrite of an existing dynamic configuration unconditionally:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ curl -s -XPUT -d \
|
||||
'{"maximum_lag_on_failover":1048576,"retry_timeout":10,"postgresql":{"use_slots":true,"use_pg_rewind":true,"parameters":{"hot_standby":"on","wal_log_hints":"on","wal_keep_segments":8,"wal_level":"hot_standby","unix_socket_directories":".","max_wal_senders":5}},"loop_wait":3,"ttl":20}' \
|
||||
http://localhost:8008/config | jq .
|
||||
{
|
||||
"ttl": 20,
|
||||
"maximum_lag_on_failover": 1048576,
|
||||
"retry_timeout": 10,
|
||||
"postgresql": {
|
||||
"use_slots": true,
|
||||
"parameters": {
|
||||
"hot_standby": "on",
|
||||
"unix_socket_directories": ".",
|
||||
"wal_keep_segments": 8,
|
||||
"wal_level": "hot_standby",
|
||||
"wal_log_hints": "on",
|
||||
"max_wal_senders": 5
|
||||
},
|
||||
"use_pg_rewind": true
|
||||
},
|
||||
"loop_wait": 3
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
### confd
|
||||
|
||||
`confd` directory contains haproxy template files for the [confd](https://github.com/kelseyhightower/confd) -- lightweight configuration management tool
|
||||
You need to copy content of `confd` directory into /etcd/confd and run confd service:
|
||||
```bash
|
||||
$ confd -prefix=/service/$PATRONI_SCOPE -backend etcd -node $PATRONI_ETCD_HOST -interval=10
|
||||
```
|
||||
It will periodically update haproxy.cfg with the actual list of Patroni nodes from `etcd` and "reload" haproxy when it is necessary.
|
||||
|
||||
|
||||
### startup-scripts
|
||||
|
||||
`startup-scripts` directory contains startup scripts for various OSes and management tools for Patroni.
|
||||
@@ -1,13 +0,0 @@
|
||||
[template]
|
||||
#prefix = "/service/batman"
|
||||
#owner = "haproxy"
|
||||
#mode = "0644"
|
||||
src = "haproxy.tmpl"
|
||||
dest = "/etc/haproxy/haproxy.cfg"
|
||||
|
||||
check_cmd = "/usr/sbin/haproxy -c -f {{ .src }}"
|
||||
reload_cmd = "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D -sf $(cat /var/run/haproxy.pid)"
|
||||
|
||||
keys = [
|
||||
"/members/",
|
||||
]
|
||||
@@ -1,28 +0,0 @@
|
||||
global
|
||||
maxconn 100
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode tcp
|
||||
retries 2
|
||||
timeout client 30m
|
||||
timeout connect 4s
|
||||
timeout server 30m
|
||||
timeout check 5s
|
||||
|
||||
frontend master_postgresql
|
||||
bind *:5000
|
||||
default_backend backend_master
|
||||
|
||||
frontend replicas_postgresql
|
||||
bind *:5001
|
||||
default_backend backend_replicas
|
||||
|
||||
backend backend_master
|
||||
option httpchk OPTIONS /master
|
||||
{{range gets "/members/*"}} server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}
|
||||
{{end}}
|
||||
backend backend_replicas
|
||||
option httpchk OPTIONS /replica
|
||||
{{range gets "/members/*"}} server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}
|
||||
{{end}}
|
||||
@@ -8,6 +8,3 @@ Scripts supplied:
|
||||
### patroni.upstart.conf
|
||||
|
||||
Upstart job for Ubuntu 12.04 or 14.04. Requires Upstart > 1.4. Intended for systems where Patroni has been installed on a base system, rather than in Docker.
|
||||
|
||||
### patroni.service
|
||||
Systemd service file, to be copied to /etc/systemd/system/patroni.service, tested on Centos 7.1 with Patroni installed from pip.
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
# This is an example systemd config file for Patroni
|
||||
# You can copy it to "/etc/systemd/system/patroni.service",
|
||||
|
||||
[Unit]
|
||||
Description=Runners to orchestrate a high-availability PostgreSQL
|
||||
After=syslog.target network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
|
||||
User=postgres
|
||||
Group=postgres
|
||||
|
||||
# Where to send early-startup messages from the server
|
||||
# This is normally controlled by the global default set by systemd
|
||||
# StandardOutput=syslog
|
||||
|
||||
ExecStart=/bin/patroni /etc/patroni.yml
|
||||
|
||||
# only kill the patroni process, not it's children, so it will gracefully stop postgres
|
||||
KillMode=process
|
||||
|
||||
# Give a reasonable amount of time for the server to start up/shut down
|
||||
TimeoutSec=10
|
||||
|
||||
# Do not restart the service if it crashes, we want to manually inspect database on failure
|
||||
Restart=no
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
Feature: basic replication
|
||||
We should check that the basic bootstrapping, replication and failover works.
|
||||
|
||||
Scenario: check replication of a single table
|
||||
Given I start postgres0
|
||||
And postgres0 is a leader after 10 seconds
|
||||
And I start postgres1
|
||||
When I add the table foo to postgres0
|
||||
Then table foo is present on postgres1 after 20 seconds
|
||||
|
||||
Scenario: check the basic failover
|
||||
When I kill postgres0
|
||||
Then postgres1 role is the primary after 32 seconds
|
||||
When I start postgres0
|
||||
Then postgres0 role is the secondary after 20 seconds
|
||||
When I add the table bar to postgres1
|
||||
Then table bar is present on postgres0 after 20 seconds
|
||||
@@ -1,13 +0,0 @@
|
||||
Feature: cascading replication
|
||||
We should check that patroni can do base backup and streaming from the replica
|
||||
|
||||
Scenario: check a base backup and streaming replication from a replica
|
||||
Given I start postgres0
|
||||
And postgres0 is a leader after 10 seconds
|
||||
And I configure and start postgres1 with a tag clonefrom true
|
||||
And replication works from postgres0 to postgres1 after 20 seconds
|
||||
And I create label with "postgres0" in postgres0 data directory
|
||||
And I create label with "postgres1" in postgres1 data directory
|
||||
And I configure and start postgres2 with a tag replicatefrom postgres1
|
||||
Then replication works from postgres0 to postgres2 after 30 seconds
|
||||
And there is a label with "postgres1" in postgres2 data directory
|
||||
@@ -1,413 +0,0 @@
|
||||
import abc
|
||||
import consul
|
||||
import etcd
|
||||
import kazoo.client
|
||||
import kazoo.exceptions
|
||||
import os
|
||||
import psycopg2
|
||||
import shutil
|
||||
import six
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import yaml
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
class AbstractController(object):
|
||||
|
||||
def __init__(self, name, work_directory, output_dir):
|
||||
self._name = name
|
||||
self._work_directory = work_directory
|
||||
self._output_dir = output_dir
|
||||
self._handle = None
|
||||
self._log = None
|
||||
|
||||
def _has_started(self):
|
||||
return self._handle and self._handle.pid and self._handle.poll() is None
|
||||
|
||||
def _is_running(self):
|
||||
return self._has_started()
|
||||
|
||||
@abc.abstractmethod
|
||||
def _is_accessible(self):
|
||||
"""process is accessible for queries"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def _start(self):
|
||||
"""start process"""
|
||||
|
||||
def start(self, max_wait_limit=5):
|
||||
if self._is_running():
|
||||
return True
|
||||
|
||||
self._log = open(os.path.join(self._output_dir, self._name + '.log'), 'a')
|
||||
self._handle = self._start()
|
||||
|
||||
assert self._has_started(), "Process {0} is not running after being started".format(self._name)
|
||||
|
||||
for _ in range(max_wait_limit):
|
||||
if self._is_accessible():
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
assert False,\
|
||||
"{0} instance is not available for queries after {1} seconds".format(self._name, max_wait_limit)
|
||||
|
||||
def stop(self, kill=False, timeout=15):
|
||||
term = False
|
||||
start_time = time.time()
|
||||
|
||||
while self._handle and self._is_running():
|
||||
if kill:
|
||||
self._handle.kill()
|
||||
elif not term:
|
||||
self._handle.terminate()
|
||||
term = True
|
||||
time.sleep(1)
|
||||
if not kill and time.time() - start_time > timeout:
|
||||
kill = True
|
||||
|
||||
if self._log:
|
||||
self._log.close()
|
||||
|
||||
|
||||
class PatroniController(AbstractController):
|
||||
__PORT = 5440
|
||||
PATRONI_CONFIG = '{}.yml'
|
||||
""" starts and stops individual patronis"""
|
||||
|
||||
def __init__(self, dcs, name, work_directory, output_dir, tags=None):
|
||||
super(PatroniController, self).__init__('patroni_' + name, work_directory, output_dir)
|
||||
PatroniController.__PORT += 1
|
||||
self._data_dir = os.path.join(work_directory, 'data', name)
|
||||
self._connstring = None
|
||||
self._config = self._make_patroni_test_config(name, dcs, tags)
|
||||
|
||||
self._conn = None
|
||||
self._curs = None
|
||||
|
||||
def write_label(self, content):
|
||||
with open(os.path.join(self._data_dir, 'label'), 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
def read_label(self):
|
||||
try:
|
||||
with open(os.path.join(self._data_dir, 'label'), 'r') as f:
|
||||
return f.read().strip()
|
||||
except IOError:
|
||||
return None
|
||||
|
||||
def add_tag_to_config(self, tag, value):
|
||||
with open(self._config) as r:
|
||||
config = yaml.safe_load(r)
|
||||
config['tags']['tag'] = value
|
||||
with open(self._config, 'w') as w:
|
||||
yaml.safe_dump(config, w, default_flow_style=False)
|
||||
|
||||
def _start(self):
|
||||
return subprocess.Popen(['coverage', 'run', '--source=patroni', '-p', 'patroni.py', self._config],
|
||||
stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory)
|
||||
|
||||
def _is_accessible(self):
|
||||
return self.query("SELECT 1", fail_ok=True) is not None
|
||||
|
||||
def _make_patroni_test_config(self, name, dcs, tags):
|
||||
patroni_config_name = self.PATRONI_CONFIG.format(name)
|
||||
patroni_config_path = os.path.join(self._output_dir, patroni_config_name)
|
||||
|
||||
with open(patroni_config_name) as f:
|
||||
config = yaml.safe_load(f)
|
||||
config.pop('etcd')
|
||||
|
||||
host = config['postgresql']['listen'].split(':')[0]
|
||||
|
||||
config['postgresql']['listen'] = config['postgresql']['connect_address'] = '{0}:{1}'.format(host, self.__PORT)
|
||||
|
||||
user = config['postgresql'].get('authentication', config['postgresql']).get('superuser', {})
|
||||
self._connkwargs = {k: user[n] for n, k in [('username', 'user'), ('password', 'password')] if n in user}
|
||||
self._connkwargs.update({'host': host, 'port': self.__PORT, 'database': 'postgres'})
|
||||
|
||||
config['name'] = name
|
||||
config['postgresql']['data_dir'] = self._data_dir
|
||||
config['postgresql']['parameters'].update({
|
||||
'logging_collector': 'on', 'log_destination': 'csvlog', 'log_directory': self._output_dir,
|
||||
'log_filename': name + '.log', 'log_statement': 'all', 'log_min_messages': 'debug1'})
|
||||
|
||||
if 'bootstrap' in config and 'initdb' in config['bootstrap']:
|
||||
config['bootstrap']['initdb'].extend([{'auth': 'md5'}, {'auth-host': 'md5'}])
|
||||
|
||||
if tags:
|
||||
config['tags'] = tags
|
||||
|
||||
with open(patroni_config_path, 'w') as f:
|
||||
yaml.safe_dump(config, f, default_flow_style=False)
|
||||
|
||||
return patroni_config_path
|
||||
|
||||
def _connection(self):
|
||||
if not self._conn or self._conn.closed != 0:
|
||||
self._conn = psycopg2.connect(**self._connkwargs)
|
||||
self._conn.autocommit = True
|
||||
return self._conn
|
||||
|
||||
def _cursor(self):
|
||||
if not self._curs or self._curs.closed or self._curs.connection.closed != 0:
|
||||
self._curs = self._connection().cursor()
|
||||
return self._curs
|
||||
|
||||
def query(self, query, fail_ok=False):
|
||||
try:
|
||||
cursor = self._cursor()
|
||||
cursor.execute(query)
|
||||
return cursor
|
||||
except psycopg2.Error:
|
||||
if not fail_ok:
|
||||
raise
|
||||
|
||||
def check_role_has_changed_to(self, new_role, timeout=10):
|
||||
bound_time = time.time() + timeout
|
||||
recovery_status = new_role != 'primary'
|
||||
while time.time() < bound_time:
|
||||
cur = self.query("SELECT pg_is_in_recovery()", fail_ok=True)
|
||||
if cur:
|
||||
row = cur.fetchone()
|
||||
if row and row[0] == recovery_status:
|
||||
return True
|
||||
time.sleep(1)
|
||||
return False
|
||||
|
||||
|
||||
class AbstractDcsController(AbstractController):
|
||||
|
||||
_CLUSTER_NODE = '/service/batman'
|
||||
|
||||
def _is_accessible(self):
|
||||
return self._is_running()
|
||||
|
||||
def stop_and_remove_work_directory(self, timeout=15):
|
||||
""" terminate process and wipe out the temp work directory, but only if we actually started it"""
|
||||
self.stop(timeout=timeout)
|
||||
if self._work_directory:
|
||||
shutil.rmtree(self._work_directory)
|
||||
|
||||
def path(self, key=None):
|
||||
return self._CLUSTER_NODE + (key and '/' + key or '')
|
||||
|
||||
@abc.abstractmethod
|
||||
def query(self, key):
|
||||
""" query for a value of a given key """
|
||||
|
||||
@abc.abstractmethod
|
||||
def set(self, key, value):
|
||||
""" set a value to a given key """
|
||||
|
||||
@abc.abstractmethod
|
||||
def cleanup_service_tree(self):
|
||||
""" clean all contents stored in the tree used for the tests """
|
||||
|
||||
|
||||
class ConsulController(AbstractDcsController):
|
||||
|
||||
def __init__(self, output_dir):
|
||||
super(ConsulController, self).__init__('consul', tempfile.mkdtemp(), output_dir)
|
||||
os.environ['PATRONI_CONSUL_HOST'] = 'localhost:8500'
|
||||
self._client = consul.Consul()
|
||||
|
||||
def _start(self):
|
||||
return subprocess.Popen(['consul', 'agent', '-server', '-bootstrap', '-advertise=127.0.0.1',
|
||||
'-data-dir', self._work_directory], stdout=self._log, stderr=subprocess.STDOUT)
|
||||
|
||||
def _is_running(self):
|
||||
try:
|
||||
return bool(self._client.status.leader())
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def path(self, key=None):
|
||||
return super(ConsulController, self).path(key)[1:]
|
||||
|
||||
def query(self, key):
|
||||
_, value = self._client.kv.get(self.path(key))
|
||||
return value and value['Value'].decode('utf-8')
|
||||
|
||||
def set(self, key, value):
|
||||
self._client.kv.put(self.path(key), value)
|
||||
|
||||
def cleanup_service_tree(self):
|
||||
self._client.kv.delete(self.path(), recurse=True)
|
||||
|
||||
|
||||
class EtcdController(AbstractDcsController):
|
||||
|
||||
""" handles all etcd related tasks, used for the tests setup and cleanup """
|
||||
|
||||
def __init__(self, output_dir):
|
||||
super(EtcdController, self).__init__('etcd', tempfile.mkdtemp(), output_dir)
|
||||
os.environ['PATRONI_ETCD_HOST'] = 'localhost:4001'
|
||||
self._client = etcd.Client()
|
||||
|
||||
def _start(self):
|
||||
return subprocess.Popen(["etcd", "--debug", "--data-dir", self._work_directory],
|
||||
stdout=self._log, stderr=subprocess.STDOUT)
|
||||
|
||||
def query(self, key):
|
||||
try:
|
||||
return self._client.get(self.path(key)).value
|
||||
except etcd.EtcdKeyNotFound:
|
||||
return None
|
||||
|
||||
def set(self, key, value):
|
||||
self._client.set(self.path(key), value)
|
||||
|
||||
def cleanup_service_tree(self):
|
||||
try:
|
||||
self._client.delete(self.path(), recursive=True)
|
||||
except (etcd.EtcdKeyNotFound, etcd.EtcdConnectionFailed):
|
||||
return
|
||||
except Exception as e:
|
||||
assert False, "exception when cleaning up etcd contents: {0}".format(e)
|
||||
|
||||
def _is_running(self):
|
||||
# if etcd is running, but we didn't start it
|
||||
try:
|
||||
return bool(self._client.machines)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class ZooKeeperController(AbstractDcsController):
|
||||
|
||||
""" handles all zookeeper related tasks, used for the tests setup and cleanup """
|
||||
|
||||
def __init__(self, output_dir, export_env=True):
|
||||
super(ZooKeeperController, self).__init__('zookeeper', None, output_dir)
|
||||
if export_env:
|
||||
os.environ['PATRONI_ZOOKEEPER_HOSTS'] = "'localhost:2181'"
|
||||
self._client = kazoo.client.KazooClient()
|
||||
|
||||
def _start(self):
|
||||
pass # TODO: implement later
|
||||
|
||||
def query(self, key):
|
||||
try:
|
||||
return self._client.get(self.path(key))[0].decode('utf-8')
|
||||
except kazoo.exceptions.NoNodeError:
|
||||
return None
|
||||
|
||||
def set(self, key, value):
|
||||
self._client.set(self.path(key), value.encode('utf-8'))
|
||||
|
||||
def cleanup_service_tree(self):
|
||||
try:
|
||||
self._client.delete(self.path(), recursive=True)
|
||||
except (kazoo.exceptions.NoNodeError):
|
||||
return
|
||||
except Exception as e:
|
||||
assert False, "exception when cleaning up zookeeper contents: {0}".format(e)
|
||||
|
||||
def _is_running(self):
|
||||
# if zookeeper is running, but we didn't start it
|
||||
if self._client.connected:
|
||||
return True
|
||||
try:
|
||||
return self._client.start(1) or True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class ExhibitorController(ZooKeeperController):
|
||||
|
||||
def __init__(self, output_dir):
|
||||
super(ExhibitorController, self).__init__(output_dir, False)
|
||||
os.environ.update({'PATRONI_EXHIBITOR_HOSTS': 'localhost', 'PATRONI_EXHIBITOR_PORT': '8181'})
|
||||
|
||||
|
||||
class PatroniPoolController(object):
|
||||
|
||||
KNOWN_DCS = {'consul': ConsulController, 'etcd': EtcdController,
|
||||
'zookeeper': ZooKeeperController, 'exhibitor': ExhibitorController}
|
||||
|
||||
def __init__(self):
|
||||
self._dcs = None
|
||||
self._output_dir = None
|
||||
self._patroni_path = None
|
||||
self._processes = {}
|
||||
self.create_and_set_output_directory('')
|
||||
|
||||
@property
|
||||
def patroni_path(self):
|
||||
if self._patroni_path is None:
|
||||
cwd = os.path.realpath(__file__)
|
||||
while True:
|
||||
cwd, entry = os.path.split(cwd)
|
||||
if entry == 'features' or cwd == '/':
|
||||
break
|
||||
self._patroni_path = cwd
|
||||
return self._patroni_path
|
||||
|
||||
@property
|
||||
def output_dir(self):
|
||||
return self._output_dir
|
||||
|
||||
def start(self, pg_name, max_wait_limit=20, tags=None):
|
||||
if pg_name not in self._processes:
|
||||
self._processes[pg_name] = PatroniController(self.dcs, pg_name, self.patroni_path, self._output_dir, tags)
|
||||
self._processes[pg_name].start(max_wait_limit)
|
||||
|
||||
def __getattr__(self, func):
|
||||
if func not in ['stop', 'query', 'write_label', 'read_label', 'check_role_has_changed_to', 'add_tag_to_config']:
|
||||
raise AttributeError("PatroniPoolController instance has no attribute '{0}'".format(func))
|
||||
|
||||
def wrapper(pg_name, *args, **kwargs):
|
||||
return getattr(self._processes[pg_name], func)(*args, **kwargs)
|
||||
return wrapper
|
||||
|
||||
def stop_all(self):
|
||||
for ctl in self._processes.values():
|
||||
ctl.stop()
|
||||
self._processes.clear()
|
||||
|
||||
def create_and_set_output_directory(self, feature_name):
|
||||
feature_dir = os.path.join(self.patroni_path, 'features/output', feature_name.replace(' ', '_'))
|
||||
if os.path.exists(feature_dir):
|
||||
shutil.rmtree(feature_dir)
|
||||
os.makedirs(feature_dir)
|
||||
self._output_dir = feature_dir
|
||||
|
||||
@property
|
||||
def dcs(self):
|
||||
if self._dcs is None:
|
||||
self._dcs = os.environ.pop('DCS', 'etcd')
|
||||
assert self._dcs in self.KNOWN_DCS, 'Unsupported dcs: ' + self._dcs
|
||||
return self._dcs
|
||||
|
||||
|
||||
# actions to execute on start/stop of the tests and before running invidual features
|
||||
def before_all(context):
|
||||
context.pctl = PatroniPoolController()
|
||||
context.dcs_ctl = context.pctl.KNOWN_DCS[context.pctl.dcs](context.pctl.output_dir)
|
||||
context.dcs_ctl.start()
|
||||
try:
|
||||
context.dcs_ctl.cleanup_service_tree()
|
||||
except AssertionError: # after_all handlers won't be executed in before_all
|
||||
context.dcs_ctl.stop_and_remove_work_directory()
|
||||
raise
|
||||
|
||||
|
||||
def after_all(context):
|
||||
context.dcs_ctl.stop_and_remove_work_directory()
|
||||
subprocess.call(['coverage', 'combine'])
|
||||
subprocess.call(['coverage', 'report'])
|
||||
|
||||
|
||||
def before_feature(context, feature):
|
||||
""" create per-feature output directory to collect Patroni and PostgreSQL logs """
|
||||
context.pctl.create_and_set_output_directory(feature.name)
|
||||
|
||||
|
||||
def after_feature(context, feature):
|
||||
""" stop all Patronis, remove their data directory and cleanup the keys in etcd """
|
||||
context.pctl.stop_all()
|
||||
shutil.rmtree(os.path.join(context.pctl.patroni_path, 'data'))
|
||||
context.dcs_ctl.cleanup_service_tree()
|
||||
@@ -1,79 +0,0 @@
|
||||
Feature: patroni api
|
||||
We should check that patroni correctly responds to valid and not-valid API requests.
|
||||
|
||||
Scenario: check API requests on a stand-alone server
|
||||
Given I start postgres0
|
||||
And postgres0 is a leader after 10 seconds
|
||||
When I issue a GET request to http://127.0.0.1:8008/
|
||||
Then I receive a response code 200
|
||||
And I receive a response state running
|
||||
And I receive a response role master
|
||||
When I issue a GET request to http://127.0.0.1:8008/replica
|
||||
Then I receive a response code 503
|
||||
When I run patronictl.py reinit batman postgres0 --force
|
||||
Then I receive a response returncode 0
|
||||
And I receive a response output "reinitialize failed for member postgres0, status code=503, (I am the leader, can not reinitialize)"
|
||||
When I run patronictl.py failover batman --master postgres0 --force
|
||||
Then I receive a response returncode 1
|
||||
And I receive a response output "Error: No candidates found to failover to"
|
||||
When I issue a POST request to http://127.0.0.1:8008/failover with {"leader": "postgres0"}
|
||||
Then I receive a response code 500
|
||||
And I receive a response text failover is not possible: cluster does not have members except leader
|
||||
When I issue an empty POST request to http://127.0.0.1:8008/failover
|
||||
Then I receive a response code 400
|
||||
When I issue a POST request to http://127.0.0.1:8008/failover with {"foo": "bar"}
|
||||
Then I receive a response code 400
|
||||
And I receive a response text "No values given for required parameters leader and candidate"
|
||||
|
||||
Scenario: check local configuration reload
|
||||
Given I issue an empty POST request to http://127.0.0.1:8008/reload
|
||||
Then I receive a response code 200
|
||||
And I receive a response text nothing changed
|
||||
When I add tag new_tag new_value to postgres0 config
|
||||
And I issue an empty POST request to http://127.0.0.1:8008/reload
|
||||
Then I receive a response code 202
|
||||
|
||||
Scenario: check dynamic configuration change via DCS
|
||||
Given I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "loop_wait": 1, "postgresql": {"parameters": {"max_connections": 101}}}
|
||||
Then I receive a response code 200
|
||||
And I receive a response loop_wait 1
|
||||
And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 11 seconds
|
||||
When I issue a GET request to http://127.0.0.1:8008/config
|
||||
Then I receive a response code 200
|
||||
And I receive a response loop_wait 1
|
||||
When I issue a GET request to http://127.0.0.1:8008/patroni
|
||||
Then I receive a response code 200
|
||||
And I receive a response tags {'tag': 'new_value'}
|
||||
|
||||
Scenario: check API requests for the primary-replica pair
|
||||
Given I start postgres1
|
||||
And replication works from postgres0 to postgres1 after 20 seconds
|
||||
When I issue a GET request to http://127.0.0.1:8009/replica
|
||||
Then I receive a response code 200
|
||||
And I receive a response state running
|
||||
And I receive a response role replica
|
||||
When I run patronictl.py reinit batman postgres1 --force
|
||||
Then I receive a response returncode 0
|
||||
And I receive a response output "Succesful reinitialize on member postgres1"
|
||||
When I run patronictl.py restart batman postgres0 --force
|
||||
Then I receive a response returncode 0
|
||||
And I receive a response output "Succesful restart on member postgres0"
|
||||
And postgres0 role is the primary after 5 seconds
|
||||
When I sleep for 10 seconds
|
||||
Then postgres1 role is the secondary after 15 seconds
|
||||
|
||||
Scenario: check the failover via the API
|
||||
Given I run patronictl.py failover batman --master postgres0 --candidate postgres1 --force
|
||||
Then I receive a response returncode 0
|
||||
And postgres1 is a leader after 5 seconds
|
||||
And postgres1 role is the primary after 5 seconds
|
||||
And postgres0 role is the secondary after 10 seconds
|
||||
And replication works from postgres1 to postgres0 after 20 seconds
|
||||
|
||||
Scenario: check the scheduled failover
|
||||
Given I issue a scheduled failover from postgres1 to postgres0 in 1 seconds
|
||||
Then I receive a response returncode 0
|
||||
And postgres0 is a leader after 20 seconds
|
||||
And postgres0 role is the primary after 5 seconds
|
||||
And postgres1 role is the secondary after 10 seconds
|
||||
And replication works from postgres0 to postgres1 after 25 seconds
|
||||
@@ -1,54 +0,0 @@
|
||||
import psycopg2 as pg
|
||||
|
||||
from behave import step, then
|
||||
from time import sleep, time
|
||||
|
||||
|
||||
@step('I start {name:w}')
|
||||
def start_patroni(context, name):
|
||||
return context.pctl.start(name)
|
||||
|
||||
|
||||
@step('I shut down {name:w}')
|
||||
def stop_patroni(context, name):
|
||||
return context.pctl.stop(name)
|
||||
|
||||
|
||||
@step('I kill {name:w}')
|
||||
def kill_patroni(context, name):
|
||||
return context.pctl.stop(name, kill=True)
|
||||
|
||||
|
||||
@step('I add the table {table_name:w} to {pg_name:w}')
|
||||
def add_table(context, table_name, pg_name):
|
||||
# parse the configuration file and get the port
|
||||
try:
|
||||
context.pctl.query(pg_name, "CREATE TABLE {0}()".format(table_name))
|
||||
except pg.Error as e:
|
||||
assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e)
|
||||
|
||||
|
||||
@then('Table {table_name:w} is present on {pg_name:w} after {max_replication_delay:d} seconds')
|
||||
def table_is_present_on(context, table_name, pg_name, max_replication_delay):
|
||||
for _ in range(int(max_replication_delay)):
|
||||
if context.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
|
||||
break
|
||||
sleep(1)
|
||||
else:
|
||||
assert False,\
|
||||
"Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay)
|
||||
|
||||
|
||||
@then('{pg_name:w} role is the {pg_role:w} after {max_promotion_timeout:d} seconds')
|
||||
def check_role(context, pg_name, pg_role, max_promotion_timeout):
|
||||
assert context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)),\
|
||||
"{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
|
||||
|
||||
|
||||
@step('replication works from {master:w} to {replica:w} after {time_limit:d} seconds')
|
||||
@then('replication works from {master:w} to {replica:w} after {time_limit:d} seconds')
|
||||
def replication_works(context, master, replica, time_limit):
|
||||
context.execute_steps(u"""
|
||||
When I add the table test_{0} to {1}
|
||||
Then table test_{0} is present on {2} after {3} seconds
|
||||
""".format(int(time()), master, replica, time_limit))
|
||||
@@ -1,17 +0,0 @@
|
||||
from behave import step, then
|
||||
|
||||
|
||||
@step('I configure and start {name:w} with a tag {tag_name:w} {tag_value:w}')
|
||||
def start_patroni_with_a_name_value_tag(context, name, tag_name, tag_value):
|
||||
return context.pctl.start(name, tags={tag_name: tag_value})
|
||||
|
||||
|
||||
@then('There is a label with "{content:w}" in {name:w} data directory')
|
||||
def check_label(context, content, name):
|
||||
label = context.pctl.read_label(name)
|
||||
assert label == content, "{0} is not equal to {1}".format(label, content)
|
||||
|
||||
|
||||
@step('I create label with "{content:w}" in {name:w} data directory')
|
||||
def write_label(context, content, name):
|
||||
context.pctl.write_label(name, content)
|
||||
@@ -1,138 +0,0 @@
|
||||
import json
|
||||
import parse
|
||||
import pytz
|
||||
import requests
|
||||
import shlex
|
||||
import subprocess
|
||||
import time
|
||||
import yaml
|
||||
|
||||
from behave import register_type, step, then
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
|
||||
@parse.with_pattern(r'https?://(?:\w|\.|:|/)+')
|
||||
def parse_url(text):
|
||||
return text
|
||||
|
||||
|
||||
register_type(url=parse_url)
|
||||
|
||||
|
||||
# there is no way we can find out if the node has already
|
||||
# started as a leader without checking the DCS. We cannot
|
||||
# just rely on the database availability, since there is
|
||||
# a short gap between the time PostgreSQL becomes available
|
||||
# and Patroni assuming the leader role.
|
||||
@step('{name:w} is a leader after {time_limit:d} seconds')
|
||||
@then('{name:w} is a leader after {time_limit:d} seconds')
|
||||
def is_a_leader(context, name, time_limit):
|
||||
max_time = time.time() + int(time_limit)
|
||||
while (context.dcs_ctl.query("leader") != name):
|
||||
time.sleep(1)
|
||||
assert time.time() < max_time, "{0} is not a leader in dcs after {1} seconds".format(name, time_limit)
|
||||
|
||||
|
||||
@step('I sleep for {value:d} seconds')
|
||||
def sleep_for_n_seconds(context, value):
|
||||
time.sleep(int(value))
|
||||
|
||||
|
||||
def _set_response(context, response):
|
||||
context.status_code = response.status_code
|
||||
data = response.content.decode('utf-8')
|
||||
ct = response.headers.get('content-type', '')
|
||||
if ct.startswith('application/json') or\
|
||||
ct.startswith('text/yaml') or\
|
||||
ct.startswith('text/x-yaml') or\
|
||||
ct.startswith('application/yaml') or\
|
||||
ct.startswith('application/x-yaml'):
|
||||
try:
|
||||
context.response = yaml.safe_load(data)
|
||||
except ValueError:
|
||||
context.response = data
|
||||
else:
|
||||
context.response = data
|
||||
|
||||
|
||||
@step('I issue a GET request to {url:url}')
|
||||
def do_get(context, url):
|
||||
try:
|
||||
r = requests.get(url)
|
||||
except requests.exceptions.RequestException:
|
||||
context.status_code = None
|
||||
context.response = None
|
||||
else:
|
||||
_set_response(context, r)
|
||||
|
||||
|
||||
@step('I issue an empty POST request to {url:url}')
|
||||
def do_post_empty(context, url):
|
||||
do_request(context, 'POST', url, None)
|
||||
|
||||
|
||||
@step('I issue a {request_method:w} request to {url:url} with {data}')
|
||||
def do_request(context, request_method, url, data):
|
||||
data = data and json.loads(data) or {}
|
||||
try:
|
||||
if request_method == 'PATCH':
|
||||
r = requests.patch(url, json=data)
|
||||
else:
|
||||
r = requests.post(url, json=data)
|
||||
except requests.exceptions.RequestException:
|
||||
context.status_code = None
|
||||
context.response = None
|
||||
else:
|
||||
_set_response(context, r)
|
||||
|
||||
|
||||
@step('I run {cmd}')
|
||||
def do_run(context, cmd):
|
||||
cmd = ['coverage', 'run', '--source=patroni', '-p'] + shlex.split(cmd)
|
||||
try:
|
||||
response = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
|
||||
context.status_code = 0
|
||||
except subprocess.CalledProcessError as e:
|
||||
response = e.output
|
||||
context.status_code = e.returncode
|
||||
context.response = response.decode('utf-8').strip()
|
||||
|
||||
|
||||
@then('I receive a response {component:w} {data}')
|
||||
def check_response(context, component, data):
|
||||
if component == 'code':
|
||||
assert context.status_code == int(data),\
|
||||
"status code {0} != {1}, response: {2}".format(context.status_code, data, context.response)
|
||||
elif component == 'returncode':
|
||||
assert context.status_code == int(data), "return code {0} != {1}".format(context.status_code, data)
|
||||
elif component == 'text':
|
||||
assert context.response == data.strip('"'), "response {0} does not contain {1}".format(context.response, data)
|
||||
elif component == 'output':
|
||||
assert data.strip('"') in context.response, "response {0} does not contain {1}".format(context.response, data)
|
||||
else:
|
||||
assert component in context.response, "{0} is not part of the response".format(component)
|
||||
assert str(context.response[component]) == str(data), "{0} does not contain {1}".format(component, data)
|
||||
|
||||
|
||||
@step('I issue a scheduled failover from {from_host:w} to {to_host:w} in {in_seconds:d} seconds')
|
||||
def scheduled_failover(context, from_host, to_host, in_seconds):
|
||||
context.execute_steps(u"""
|
||||
Given I run patronictl.py failover batman --master {0} --candidate {1} --scheduled "{2}" --force
|
||||
""".format(from_host, to_host, datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds))))
|
||||
|
||||
|
||||
@step('I add tag {tag:w} {value:w} to {pg_name:w} config')
|
||||
def add_tag_to_config(context, tag, value, pg_name):
|
||||
context.pctl.add_tag_to_config(pg_name, tag, value)
|
||||
|
||||
|
||||
@then('Response on GET {url} contains {value} after {timeout:d} seconds')
|
||||
def check_http_response(context, url, value, timeout):
|
||||
for _ in range(int(timeout)):
|
||||
r = requests.get(url)
|
||||
if value in r.content.decode('utf-8'):
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
assert False,\
|
||||
"Value {0} is not present in response after {1} seconds".format(value, timeout)
|
||||
+31
-85
@@ -1,84 +1,42 @@
|
||||
import logging
|
||||
import signal
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import yaml
|
||||
|
||||
from patroni.api import RestApiServer
|
||||
from patroni.config import Config
|
||||
from patroni.dcs import get_dcs
|
||||
from patroni.exceptions import DCSError
|
||||
from patroni.etcd import Etcd
|
||||
from patroni.ha import Ha
|
||||
from patroni.postgresql import Postgresql
|
||||
from patroni.utils import reap_children, sigchld_handler
|
||||
from patroni.version import __version__
|
||||
from patroni.utils import setup_signal_handlers, reap_children
|
||||
from patroni.zookeeper import ZooKeeper
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Patroni(object):
|
||||
class Patroni:
|
||||
|
||||
def __init__(self):
|
||||
self.setup_signal_handlers()
|
||||
|
||||
self.version = __version__
|
||||
self.config = Config()
|
||||
self.dcs = get_dcs(self.config)
|
||||
self.load_dynamic_configuration()
|
||||
|
||||
self.postgresql = Postgresql(self.config['postgresql'])
|
||||
self.api = RestApiServer(self, self.config['restapi'])
|
||||
def __init__(self, config):
|
||||
self.nap_time = config['loop_wait']
|
||||
self.tags = config.get('tags', dict())
|
||||
self.bdr = config.get('bdr', dict())
|
||||
self.postgresql = Postgresql(config['postgresql'], self.bdr)
|
||||
self.dcs = self.get_dcs(self.postgresql.name, config)
|
||||
self.api = RestApiServer(self, config['restapi'])
|
||||
self.ha = Ha(self)
|
||||
|
||||
self.tags = self.get_tags()
|
||||
self.nap_time = self.config['loop_wait']
|
||||
self.next_run = time.time()
|
||||
|
||||
def load_dynamic_configuration(self):
|
||||
while True:
|
||||
try:
|
||||
cluster = self.dcs.get_cluster()
|
||||
if cluster and cluster.config:
|
||||
self.config.set_dynamic_configuration(cluster.config)
|
||||
elif not self.config.dynamic_configuration and 'bootstrap' in self.config:
|
||||
self.config.set_dynamic_configuration(self.config['bootstrap']['dcs'])
|
||||
break
|
||||
except DCSError:
|
||||
logger.warning('Can not get cluster from dcs')
|
||||
|
||||
def get_tags(self):
|
||||
return {tag: value for tag, value in self.config.get('tags', {}).items()
|
||||
if tag not in ('clonefrom', 'nofailover', 'noloadbalance') or value}
|
||||
|
||||
@property
|
||||
def nofailover(self):
|
||||
return self.tags.get('nofailover', False)
|
||||
|
||||
def reload_config(self):
|
||||
try:
|
||||
self.tags = self.get_tags()
|
||||
self.nap_time = self.config['loop_wait']
|
||||
self.dcs.set_ttl(self.config.get('ttl') or 30)
|
||||
self.dcs.set_retry_timeout(self.config.get('retry_timeout') or self.nap_time)
|
||||
self.api.reload_config(self.config['restapi'])
|
||||
self.postgresql.reload_config(self.config['postgresql'])
|
||||
except Exception:
|
||||
logger.exception('Failed to reload config_file=%s', self.config.config_file)
|
||||
|
||||
@property
|
||||
def replicatefrom(self):
|
||||
return self.tags.get('replicatefrom')
|
||||
|
||||
def sighup_handler(self, *args):
|
||||
self._received_sighup = True
|
||||
|
||||
def sigterm_handler(self, *args):
|
||||
if not self._received_sigterm:
|
||||
self._received_sigterm = True
|
||||
sys.exit()
|
||||
|
||||
@property
|
||||
def noloadbalance(self):
|
||||
return self.tags.get('noloadbalance', False)
|
||||
@staticmethod
|
||||
def get_dcs(name, config):
|
||||
if 'etcd' in config:
|
||||
return Etcd(name, config['etcd'])
|
||||
if 'zookeeper' in config:
|
||||
return ZooKeeper(name, config['zookeeper'])
|
||||
raise Exception('Can not find suitable configuration of distributed configuration store')
|
||||
|
||||
def schedule_next_run(self):
|
||||
self.next_run += self.nap_time
|
||||
@@ -93,42 +51,30 @@ class Patroni(object):
|
||||
self.api.start()
|
||||
self.next_run = time.time()
|
||||
|
||||
while not self._received_sigterm:
|
||||
if self._received_sighup:
|
||||
self._received_sighup = False
|
||||
if self.config.reload_local_configuration():
|
||||
self.reload_config()
|
||||
|
||||
while True:
|
||||
logger.info(self.ha.run_cycle())
|
||||
|
||||
cluster = self.dcs.cluster
|
||||
if cluster and cluster.config and self.config.set_dynamic_configuration(cluster.config):
|
||||
self.reload_config()
|
||||
|
||||
if not self.postgresql.data_directory_empty():
|
||||
self.config.save_cache()
|
||||
|
||||
reap_children()
|
||||
self.schedule_next_run()
|
||||
|
||||
def setup_signal_handlers(self):
|
||||
self._received_sighup = False
|
||||
self._received_sigterm = False
|
||||
signal.signal(signal.SIGHUP, self.sighup_handler)
|
||||
signal.signal(signal.SIGTERM, self.sigterm_handler)
|
||||
signal.signal(signal.SIGCHLD, sigchld_handler)
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
|
||||
logging.getLogger('requests').setLevel(logging.WARNING)
|
||||
setup_signal_handlers()
|
||||
|
||||
patroni = Patroni()
|
||||
if len(sys.argv) < 2 or not os.path.isfile(sys.argv[1]):
|
||||
print('Usage: {} config.yml'.format(sys.argv[0]))
|
||||
return
|
||||
|
||||
with open(sys.argv[1], 'r') as f:
|
||||
config = yaml.load(f)
|
||||
|
||||
patroni = Patroni(config)
|
||||
try:
|
||||
patroni.run()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
patroni.api.shutdown()
|
||||
patroni.postgresql.stop(checkpoint=False)
|
||||
patroni.postgresql.stop()
|
||||
patroni.dcs.delete_leader()
|
||||
|
||||
+115
-218
@@ -3,13 +3,11 @@ import fcntl
|
||||
import json
|
||||
import logging
|
||||
import psycopg2
|
||||
import socket
|
||||
import time
|
||||
import dateutil.parser
|
||||
import datetime
|
||||
import pytz
|
||||
|
||||
from patroni.exceptions import PostgresConnectionException
|
||||
from patroni.utils import deep_compare, patch_config, Retry, RetryFailedError
|
||||
from patroni.utils import Retry, RetryFailedError
|
||||
from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
|
||||
from six.moves.socketserver import ThreadingMixIn
|
||||
from threading import Thread
|
||||
@@ -33,43 +31,36 @@ def check_auth(func):
|
||||
|
||||
class RestApiHandler(BaseHTTPRequestHandler):
|
||||
|
||||
def _write_response(self, status_code, body, content_type='text/html', headers=None):
|
||||
self.send_response(status_code)
|
||||
headers = headers or {}
|
||||
if content_type:
|
||||
headers['Content-Type'] = content_type
|
||||
for name, value in headers.items():
|
||||
self.send_header(name, value)
|
||||
def send_auth_request(self, body):
|
||||
self.send_response(401)
|
||||
self.send_header('WWW-Authenticate', 'Basic realm=\"Patroni\"')
|
||||
self.send_header('Content-type', 'text/html')
|
||||
self.end_headers()
|
||||
self.wfile.write(body.encode('utf-8'))
|
||||
|
||||
def _write_json_response(self, status_code, response):
|
||||
self._write_response(status_code, json.dumps(response), content_type='application/json')
|
||||
|
||||
def send_auth_request(self, body):
|
||||
headers = {'WWW-Authenticate': 'Basic realm="' + self.server.patroni.__class__.__name__ + '"'}
|
||||
self._write_response(401, body, headers=headers)
|
||||
def finish(self, *args, **kwargs):
|
||||
try:
|
||||
if not self.wfile.closed:
|
||||
self.wfile.flush()
|
||||
self.wfile.close()
|
||||
except socket.error:
|
||||
pass
|
||||
self.rfile.close()
|
||||
|
||||
def check_auth_header(self):
|
||||
auth_header = self.headers.get('Authorization')
|
||||
status = self.server.check_auth_header(auth_header)
|
||||
return not status or self.send_auth_request(status)
|
||||
|
||||
def _write_status_response(self, status_code, response):
|
||||
patroni = self.server.patroni
|
||||
response.update({'tags': patroni.tags} if patroni.tags else {})
|
||||
if patroni.postgresql.sysid:
|
||||
response['database_system_identifier'] = patroni.postgresql.sysid
|
||||
if patroni.postgresql.pending_restart:
|
||||
response['pending_restart'] = True
|
||||
response['patroni'] = {'version': patroni.version, 'scope': patroni.postgresql.scope}
|
||||
self._write_json_response(status_code, response)
|
||||
def do_OPTIONS(self):
|
||||
self.do_GET(options=True)
|
||||
|
||||
def do_GET(self, write_status_code_only=False):
|
||||
def do_GET(self, options=False):
|
||||
"""Default method for processing all GET requests which can not be routed to other methods"""
|
||||
|
||||
path = '/master' if self.path == '/' else self.path
|
||||
response = self.get_postgresql_status()
|
||||
response.update(self.get_tags())
|
||||
|
||||
patroni = self.server.patroni
|
||||
cluster = patroni.dcs.cluster
|
||||
@@ -80,98 +71,48 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
status_code = 503
|
||||
elif response['role'] == 'master': # running as master but without leader lock!!!!
|
||||
status_code = 503
|
||||
elif response['role'] in path: # response['role'] != 'master'
|
||||
status_code = 503 if patroni.noloadbalance else 200
|
||||
elif response['role'] in path:
|
||||
status_code = 200
|
||||
else:
|
||||
status_code = 503
|
||||
elif 'role' in response and response['role'] in path:
|
||||
status_code = 503 if response['role'] != 'master' and patroni.noloadbalance else 200
|
||||
status_code = 200
|
||||
elif patroni.ha.restart_scheduled() and patroni.postgresql.role == 'master' and 'master' in path:
|
||||
# exceptional case for master node when the postgres is being restarted via API
|
||||
status_code = 200
|
||||
else:
|
||||
status_code = 503
|
||||
|
||||
if write_status_code_only: # when haproxy sends OPTIONS request it reads only status code and nothing more
|
||||
message = self.responses[status_code][0]
|
||||
self.wfile.write('{0} {1} {2}\r\n'.format(self.protocol_version, status_code, message).encode('utf-8'))
|
||||
else:
|
||||
self._write_status_response(status_code, response)
|
||||
|
||||
def do_OPTIONS(self):
|
||||
self.do_GET(write_status_code_only=True)
|
||||
self.send_response(status_code)
|
||||
if not options:
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps(response).encode('utf-8'))
|
||||
|
||||
def do_GET_patroni(self):
|
||||
response = self.get_postgresql_status(True)
|
||||
self._write_status_response(200, response)
|
||||
response.update(self.get_tags())
|
||||
|
||||
def do_GET_config(self):
|
||||
cluster = self.server.patroni.ha.dcs.cluster or self.server.patroni.ha.dcs.get_cluster()
|
||||
if cluster.config:
|
||||
self._write_json_response(200, cluster.config.data)
|
||||
else:
|
||||
self.send_error(502)
|
||||
|
||||
def _read_json_content(self):
|
||||
if 'content-length' not in self.headers:
|
||||
return self.send_error(411)
|
||||
try:
|
||||
content_length = int(self.headers.get('content-length'))
|
||||
request = json.loads(self.rfile.read(content_length).decode('utf-8'))
|
||||
if isinstance(request, dict) and request:
|
||||
return request
|
||||
except Exception:
|
||||
logger.exception('Bad request')
|
||||
self.send_error(400)
|
||||
|
||||
@check_auth
|
||||
def do_PATCH_config(self):
|
||||
request = self._read_json_content()
|
||||
if request:
|
||||
cluster = self.server.patroni.ha.dcs.get_cluster()
|
||||
data = cluster.config.data.copy()
|
||||
if patch_config(data, request):
|
||||
value = json.dumps(data, separators=(',', ':'))
|
||||
if not self.server.patroni.ha.dcs.set_config_value(value, cluster.config.index):
|
||||
return self.send_error(409)
|
||||
self._write_json_response(200, data)
|
||||
|
||||
@check_auth
|
||||
def do_PUT_config(self):
|
||||
request = self._read_json_content()
|
||||
if request:
|
||||
cluster = self.server.patroni.ha.dcs.get_cluster()
|
||||
if not deep_compare(request, cluster.config.data):
|
||||
value = json.dumps(request, separators=(',', ':'))
|
||||
if not self.server.patroni.ha.dcs.set_config_value(value):
|
||||
return self.send_error(502)
|
||||
self._write_json_response(200, request)
|
||||
|
||||
@check_auth
|
||||
def do_POST_reload(self):
|
||||
try:
|
||||
if self.server.patroni.config.reload_local_configuration(True):
|
||||
status_code = 202
|
||||
response = 'reload scheduled'
|
||||
self.server.patroni.sighup_handler()
|
||||
else:
|
||||
status_code = 200
|
||||
response = 'nothing changed'
|
||||
except Exception as e:
|
||||
status_code = 500
|
||||
response = str(e)
|
||||
self._write_response(status_code, response)
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps(response).encode('utf-8'))
|
||||
|
||||
@check_auth
|
||||
def do_POST_restart(self):
|
||||
status_code = 500
|
||||
data = 'restart failed'
|
||||
status_code = 503
|
||||
data = b'restart failed'
|
||||
try:
|
||||
status, data = self.server.patroni.ha.restart()
|
||||
status, msg = self.server.patroni.ha.restart()
|
||||
status_code = 200 if status else 503
|
||||
except Exception:
|
||||
data = msg.encode('utf-8')
|
||||
except:
|
||||
logger.exception('Exception during restart')
|
||||
self._write_response(status_code, data)
|
||||
|
||||
self.send_response(status_code)
|
||||
self.send_header('Content-Type', 'text/html')
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
@check_auth
|
||||
def do_POST_reinitialize(self):
|
||||
@@ -179,103 +120,73 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
cluster = ha.dcs.get_cluster()
|
||||
if cluster.is_unlocked():
|
||||
status_code = 503
|
||||
data = 'Cluster has no leader, can not reinitialize'
|
||||
data = b'Cluster has no leader, can not reinitialize'
|
||||
elif cluster.leader.name == ha.state_handler.name:
|
||||
status_code = 503
|
||||
data = 'I am the leader, can not reinitialize'
|
||||
data = b'I am the leader, can not reinitialize'
|
||||
else:
|
||||
action = ha.schedule_reinitialize()
|
||||
if action is not None:
|
||||
status_code = 503
|
||||
data = action + ' already in progress'
|
||||
data = (action + ' already in progress').encode('utf-8')
|
||||
else:
|
||||
status_code = 200
|
||||
data = 'reinitialize scheduled'
|
||||
self._write_response(status_code, data)
|
||||
data = b'reinitialize scheduled'
|
||||
|
||||
def poll_failover_result(self, leader, candidate):
|
||||
timeout = 10 if self.server.patroni.nap_time < 10 else self.server.patroni.nap_time
|
||||
for _ in range(0, timeout*2):
|
||||
self.send_response(status_code)
|
||||
self.send_header('Content-Type', 'text/html')
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
def poll_failover_result(self, leader, member):
|
||||
for a in range(0, 15):
|
||||
time.sleep(1)
|
||||
try:
|
||||
cluster = self.server.patroni.dcs.get_cluster()
|
||||
if cluster.leader and cluster.leader.name != leader:
|
||||
if not candidate or candidate == cluster.leader.name:
|
||||
return 200, 'Successfully failed over to "{0}"'.format(cluster.leader.name)
|
||||
else:
|
||||
return 200, 'Failed over to "{0}" instead of "{1}"'.format(cluster.leader.name, candidate)
|
||||
return 200, ('Successfully failed over to ' + cluster.leader.name).encode('utf-8')
|
||||
if not cluster.failover:
|
||||
return 503, 'Failover failed'
|
||||
except Exception as e:
|
||||
logger.debug('Exception occured during polling failover result: %s', e)
|
||||
return 503, 'Failover status unknown'
|
||||
return 503, b'Failover failed'
|
||||
except:
|
||||
pass
|
||||
return 503, b'Failover status unknown'
|
||||
|
||||
def is_failover_possible(self, cluster, leader, candidate):
|
||||
def is_failover_possible(self, cluster, leader, member):
|
||||
if leader and not cluster.leader or cluster.leader.name != leader:
|
||||
return 'leader name does not match'
|
||||
if candidate:
|
||||
members = [m for m in cluster.members if m.name == candidate]
|
||||
return b'leader name does not match'
|
||||
if member:
|
||||
members = [m for m in cluster.members if m.name == member]
|
||||
if not members:
|
||||
return 'candidate does not exists'
|
||||
return b'member does not exists'
|
||||
else:
|
||||
members = [m for m in cluster.members if m.name != cluster.leader.name and m.api_url]
|
||||
if not members:
|
||||
return 'failover is not possible: cluster does not have members except leader'
|
||||
for _, reachable, _, _, tags in self.server.patroni.ha.fetch_nodes_statuses(members):
|
||||
return b'failover is not possible: cluster does not have members except leader'
|
||||
for member, reachable, in_recovery, xlog_location, tags in self.server.patroni.ha.fetch_nodes_statuses(members):
|
||||
if reachable and not tags.get('nofailover', False):
|
||||
return None
|
||||
return 'failover is not possible: no good candidates have been found'
|
||||
return b'failover is not possible: no good candidates have been found'
|
||||
|
||||
@check_auth
|
||||
def do_POST_failover(self):
|
||||
request = self._read_json_content()
|
||||
if not request:
|
||||
return
|
||||
|
||||
leader = request.get('leader')
|
||||
candidate = request.get('candidate') or request.get('member')
|
||||
scheduled_at = request.get('scheduled_at')
|
||||
content_length = int(self.headers.get('content-length', 0))
|
||||
request = json.loads(self.rfile.read(content_length).decode('utf-8'))
|
||||
leader = request.get('leader', None)
|
||||
member = request.get('member', None)
|
||||
cluster = self.server.patroni.ha.dcs.get_cluster()
|
||||
status_code = 500
|
||||
|
||||
logger.info("received failover request with leader=%s candidate=%s scheduled_at=%s",
|
||||
leader, candidate, scheduled_at)
|
||||
|
||||
data = ''
|
||||
if leader or candidate:
|
||||
if scheduled_at:
|
||||
try:
|
||||
scheduled_at = dateutil.parser.parse(scheduled_at)
|
||||
if scheduled_at.tzinfo is None:
|
||||
data = 'Timezone information is mandatory for scheduled_at'
|
||||
status_code = 400
|
||||
elif scheduled_at < datetime.datetime.now(pytz.utc):
|
||||
data = 'Cannot schedule failover in the past'
|
||||
status_code = 422
|
||||
elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
|
||||
self.server.patroni.dcs.event.set()
|
||||
data = 'Failover scheduled'
|
||||
status_code = 202
|
||||
else:
|
||||
data = 'failed to write failover key into DCS'
|
||||
status_code = 503
|
||||
except (ValueError, TypeError):
|
||||
logger.exception('Invalid scheduled failover time: %s', request['scheduled_at'])
|
||||
data = 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601'
|
||||
status_code = 422
|
||||
status_code = 503
|
||||
data = self.is_failover_possible(cluster, leader, member)
|
||||
if not data:
|
||||
if not self.server.patroni.dcs.manual_failover(leader, member):
|
||||
data = b'failed to write failover key into DCS'
|
||||
else:
|
||||
data = self.is_failover_possible(cluster, leader, candidate)
|
||||
if not data:
|
||||
if self.server.patroni.dcs.manual_failover(leader, candidate):
|
||||
self.server.patroni.dcs.event.set()
|
||||
status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, candidate)
|
||||
else:
|
||||
data = 'failed to write failover key into DCS'
|
||||
status_code = 503
|
||||
else:
|
||||
status_code = 400
|
||||
data = 'No values given for required parameters leader and candidate'
|
||||
self._write_response(status_code, data)
|
||||
self.server.patroni.dcs.event.set()
|
||||
status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, member)
|
||||
|
||||
self.send_response(status_code)
|
||||
self.send_header('Content-Type', 'text/html')
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
def parse_request(self):
|
||||
"""Override parse_request method to enrich basic functionality of `BaseHTTPRequestHandler` class
|
||||
@@ -295,6 +206,12 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
self.command = mname
|
||||
return ret
|
||||
|
||||
def handle_one_request(self):
|
||||
try:
|
||||
BaseHTTPRequestHandler.handle_one_request(self)
|
||||
except socket.error:
|
||||
pass
|
||||
|
||||
def query(self, sql, *params, **kwargs):
|
||||
if not kwargs.get('retry', False):
|
||||
return self.server.query(sql, *params)
|
||||
@@ -311,18 +228,15 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
END,
|
||||
pg_xlog_location_diff(pg_last_xlog_receive_location(), '0/0')::bigint,
|
||||
pg_xlog_location_diff(pg_last_xlog_replay_location(), '0/0')::bigint,
|
||||
to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
|
||||
pg_is_in_recovery() AND pg_is_xlog_replay_paused()""", retry=retry)[0]
|
||||
return {
|
||||
'state': self.server.patroni.postgresql.state,
|
||||
'postmaster_start_time': row[0],
|
||||
'role': 'replica' if row[1] else 'master',
|
||||
'server_version': self.server.patroni.postgresql.server_version,
|
||||
'xlog': ({
|
||||
'received_location': row[3],
|
||||
'replayed_location': row[4],
|
||||
'replayed_timestamp': row[5],
|
||||
'paused': row[6]} if row[1] else {
|
||||
'paused': row[5]} if row[1] else {
|
||||
'location': row[2]
|
||||
})
|
||||
}
|
||||
@@ -331,24 +245,43 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
||||
if state == 'running':
|
||||
logger.exception('get_postgresql_status')
|
||||
state = 'unknown'
|
||||
return {'state': state, 'role': self.server.patroni.postgresql.role}
|
||||
return {'state': state}
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
logger.debug("API thread: %s - - [%s] %s", self.client_address[0], self.log_date_time_string(), fmt % args)
|
||||
def get_tags(self):
|
||||
return {'tags': self.server.patroni.tags}
|
||||
|
||||
def log_message(self, format, *args):
|
||||
logger.debug("API thread: " + format % args)
|
||||
|
||||
|
||||
class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
|
||||
|
||||
def __init__(self, patroni, config):
|
||||
self._auth_key = base64.b64encode(config['auth'].encode('utf-8')).decode('utf-8') if 'auth' in config else None
|
||||
host, port = config['listen'].split(':')
|
||||
HTTPServer.__init__(self, (host, int(port)), RestApiHandler)
|
||||
Thread.__init__(self, target=self.serve_forever)
|
||||
self._set_fd_cloexec(self.socket)
|
||||
|
||||
protocol = 'http'
|
||||
|
||||
# wrap socket with ssl if 'certfile' is defined in a config.yaml
|
||||
# Sometime it's also needed to pass reference to a 'keyfile'.
|
||||
options = {option: config[option] for option in ['certfile', 'keyfile'] if option in config}
|
||||
if options.get('certfile', None):
|
||||
import ssl
|
||||
self.socket = ssl.wrap_socket(self.socket, server_side=True, **options)
|
||||
protocol = 'https'
|
||||
|
||||
self.connection_string = '{}://{}/patroni'.format(protocol, config.get('connect_address', config['listen']))
|
||||
|
||||
self.patroni = patroni
|
||||
self.__initialize(config)
|
||||
self.__set_config_parameters(config)
|
||||
self.daemon = True
|
||||
|
||||
def query(self, sql, *params):
|
||||
cursor = None
|
||||
try:
|
||||
with self.patroni.postgresql.connection().cursor() as cursor:
|
||||
with self.patroni.postgresql.connection('postgres').cursor() as cursor:
|
||||
cursor.execute(sql, params)
|
||||
return [r for r in cursor]
|
||||
except psycopg2.Error as e:
|
||||
@@ -362,47 +295,11 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
|
||||
fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC)
|
||||
|
||||
def check_basic_auth_key(self, key):
|
||||
return self.__auth_key == key
|
||||
return self._auth_key == key
|
||||
|
||||
def check_auth_header(self, auth_header):
|
||||
if self.__auth_key:
|
||||
if self._auth_key:
|
||||
if auth_header is None:
|
||||
return 'no auth header received'
|
||||
if not auth_header.startswith('Basic ') or not self.check_basic_auth_key(auth_header[6:]):
|
||||
return 'not authenticated'
|
||||
|
||||
@staticmethod
|
||||
def __get_ssl_options(config):
|
||||
return {option: config[option] for option in ['certfile', 'keyfile'] if option in config}
|
||||
|
||||
def __set_connection_string(self, connect_address):
|
||||
self.connection_string = '{0}://{1}/patroni'.format(self.__protocol, connect_address or self.__listen)
|
||||
|
||||
def __set_config_parameters(self, config):
|
||||
self.__auth_key = base64.b64encode(config['auth'].encode('utf-8')).decode('utf-8') if 'auth' in config else None
|
||||
self.__set_connection_string(config.get('connect_address'))
|
||||
|
||||
def __initialize(self, config):
|
||||
self.__ssl_options = self.__get_ssl_options(config)
|
||||
self.__listen = config['listen']
|
||||
host, port = config['listen'].split(':')
|
||||
HTTPServer.__init__(self, (host, int(port)), RestApiHandler)
|
||||
Thread.__init__(self, target=self.serve_forever)
|
||||
self._set_fd_cloexec(self.socket)
|
||||
|
||||
self.__protocol = 'http'
|
||||
|
||||
# wrap socket with ssl if 'certfile' is defined in a config.yaml
|
||||
# Sometime it's also needed to pass reference to a 'keyfile'.
|
||||
if self.__ssl_options.get('certfile'):
|
||||
import ssl
|
||||
self.socket = ssl.wrap_socket(self.socket, server_side=True, **self.__ssl_options)
|
||||
self.__protocol = 'https'
|
||||
self.__set_connection_string(config.get('connect_address'))
|
||||
|
||||
def reload_config(self, config):
|
||||
self.__set_config_parameters(config)
|
||||
if self.__listen != config['listen'] or self.__ssl_options != self.__get_ssl_options(config):
|
||||
self.shutdown()
|
||||
self.__initialize(config)
|
||||
self.start()
|
||||
|
||||
@@ -4,9 +4,10 @@ from threading import Lock, Thread
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AsyncExecutor(object):
|
||||
class AsyncExecutor:
|
||||
|
||||
def __init__(self):
|
||||
Lock.__init__(self)
|
||||
self._busy = False
|
||||
self._thread_lock = Lock()
|
||||
self._scheduled_action = None
|
||||
@@ -50,5 +51,5 @@ class AsyncExecutor(object):
|
||||
def __enter__(self):
|
||||
self._thread_lock.acquire()
|
||||
|
||||
def __exit__(self, *args):
|
||||
def __exit__(self, type, value, traceback):
|
||||
self._thread_lock.release()
|
||||
|
||||
@@ -1,311 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import yaml
|
||||
|
||||
from collections import defaultdict
|
||||
from copy import deepcopy
|
||||
from patroni.dcs import ClusterConfig
|
||||
from patroni.postgresql import Postgresql
|
||||
from patroni.utils import deep_compare, parse_int, patch_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Config(object):
|
||||
"""
|
||||
This class is responsible for:
|
||||
|
||||
1) Building and giving access to `effective_configuration` from:
|
||||
* `Config.__DEFAULT_CONFIG` -- some sane default values
|
||||
* `dynamic_configuration` -- configuration stored in DCS
|
||||
* `local_configuration` -- configuration from `config.yml` or environment
|
||||
|
||||
2) Saving and loading `dynamic_configuration` into 'patroni.dynamic.json' file
|
||||
located in local_configuration['postgresql']['data_dir'] directory.
|
||||
This is necessary to be able to restore `dynamic_configuration`
|
||||
if DCS was accidentally wiped
|
||||
|
||||
3) Loading of configuration file in the old format and converting it into new format
|
||||
|
||||
4) Mimicking some of the `dict` interfaces to make it possible
|
||||
to work with it as with the old `config` object.
|
||||
"""
|
||||
|
||||
PATRONI_ENV_PREFIX = 'PATRONI_'
|
||||
PATRONI_CONFIG_VARIABLE = PATRONI_ENV_PREFIX + 'CONFIGURATION'
|
||||
|
||||
__CACHE_FILENAME = 'patroni.dynamic.json'
|
||||
__DEFAULT_CONFIG = {
|
||||
'ttl': 30, 'loop_wait': 10, 'retry_timeout': 10,
|
||||
'maximum_lag_on_failover': 1048576,
|
||||
'postgresql': {
|
||||
'use_slots': True,
|
||||
'parameters': {p: v[0] for p, v in Postgresql.CMDLINE_OPTIONS.items()}
|
||||
}
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self._modify_index = -1
|
||||
self._dynamic_configuration = {}
|
||||
|
||||
self.__environment_configuration = self._build_environment_configuration()
|
||||
|
||||
# Patroni reads the configuration from the command-line argument if it exists, otherwise from the environment
|
||||
self._config_file = len(sys.argv) >= 2 and os.path.isfile(sys.argv[1]) and sys.argv[1]
|
||||
if self._config_file:
|
||||
self._local_configuration = self._load_config_file()
|
||||
else:
|
||||
config_env = os.environ.pop(self.PATRONI_CONFIG_VARIABLE, None)
|
||||
self._local_configuration = config_env and yaml.safe_load(config_env) or self.__environment_configuration
|
||||
if not self._local_configuration:
|
||||
print('Usage: {0} config.yml'.format(sys.argv[0]))
|
||||
print('\tPatroni may also read the configuration from the {0} environment variable'.
|
||||
format(self.PATRONI_CONFIG_VARIABLE))
|
||||
exit(1)
|
||||
|
||||
self.__effective_configuration = self._build_effective_configuration({}, self._local_configuration)
|
||||
self._data_dir = self.__effective_configuration['postgresql']['data_dir']
|
||||
self._cache_file = os.path.join(self._data_dir, self.__CACHE_FILENAME)
|
||||
self._load_cache()
|
||||
self._cache_needs_saving = False
|
||||
|
||||
@property
|
||||
def config_file(self):
|
||||
return self._config_file
|
||||
|
||||
@property
|
||||
def dynamic_configuration(self):
|
||||
return deepcopy(self._dynamic_configuration)
|
||||
|
||||
def _load_config_file(self):
|
||||
"""Loads config.yaml from filesystem and applies some values which were set via ENV"""
|
||||
with open(self._config_file) as f:
|
||||
config = yaml.safe_load(f)
|
||||
patch_config(config, self.__environment_configuration)
|
||||
return config
|
||||
|
||||
def _load_cache(self):
|
||||
if os.path.isfile(self._cache_file):
|
||||
try:
|
||||
with open(self._cache_file) as f:
|
||||
self.set_dynamic_configuration(json.load(f))
|
||||
except Exception:
|
||||
logger.exception('Exception when loading file: %s', self._cache_file)
|
||||
|
||||
def save_cache(self):
|
||||
if self._cache_needs_saving:
|
||||
tmpfile = fd = None
|
||||
try:
|
||||
(fd, tmpfile) = tempfile.mkstemp(prefix=self.__CACHE_FILENAME, dir=self._data_dir)
|
||||
with os.fdopen(fd, 'w') as f:
|
||||
fd = None
|
||||
json.dump(self.dynamic_configuration, f)
|
||||
tmpfile = os.rename(tmpfile, self._cache_file)
|
||||
self._cache_needs_saving = False
|
||||
except Exception:
|
||||
logger.exception('Exception when saving file: %s', self._cache_file)
|
||||
if fd:
|
||||
try:
|
||||
os.close(fd)
|
||||
except Exception:
|
||||
logger.error('Can not close temporary file %s', tmpfile)
|
||||
if tmpfile and os.path.exists(tmpfile):
|
||||
try:
|
||||
os.remove(tmpfile)
|
||||
except Exception:
|
||||
logger.error('Can not remove temporary file %s', tmpfile)
|
||||
|
||||
# configuration could be either ClusterConfig or dict
|
||||
def set_dynamic_configuration(self, configuration):
|
||||
if isinstance(configuration, ClusterConfig):
|
||||
if self._modify_index == configuration.modify_index:
|
||||
return False # If the index didn't changed there is nothing to do
|
||||
self._modify_index = configuration.modify_index
|
||||
configuration = configuration.data
|
||||
|
||||
if not deep_compare(self._dynamic_configuration, configuration):
|
||||
try:
|
||||
self.__effective_configuration = self._build_effective_configuration(configuration,
|
||||
self._local_configuration)
|
||||
self._dynamic_configuration = configuration
|
||||
self._cache_needs_saving = True
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception('Exception when setting dynamic_configuration')
|
||||
|
||||
def reload_local_configuration(self, dry_run=False):
|
||||
if self.config_file:
|
||||
try:
|
||||
configuration = self._load_config_file()
|
||||
if not deep_compare(self._local_configuration, configuration):
|
||||
new_configuration = self._build_effective_configuration(self._dynamic_configuration, configuration)
|
||||
if dry_run:
|
||||
return not deep_compare(new_configuration, self.__effective_configuration)
|
||||
self._local_configuration = configuration
|
||||
self.__effective_configuration = new_configuration
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception('Exception when reloading local configuration from %s', self.config_file)
|
||||
if dry_run:
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def _process_postgresql_parameters(parameters, is_local=False):
|
||||
ret = {}
|
||||
for name, value in (parameters or {}).items():
|
||||
if name not in Postgresql.CMDLINE_OPTIONS or not is_local and Postgresql.CMDLINE_OPTIONS[name][1](value):
|
||||
ret[name] = value
|
||||
return ret
|
||||
|
||||
def _safe_copy_dynamic_configuration(self, dynamic_configuration):
|
||||
config = deepcopy(self.__DEFAULT_CONFIG)
|
||||
|
||||
for name, value in dynamic_configuration.items():
|
||||
if name == 'postgresql':
|
||||
for name, value in (value or {}).items():
|
||||
if name == 'parameters':
|
||||
config['postgresql'][name].update(self._process_postgresql_parameters(value))
|
||||
elif name not in ('connect_address', 'listen', 'data_dir', 'pgpass', 'authentication'):
|
||||
config['postgresql'][name] = deepcopy(value)
|
||||
elif name in config: # only variables present in __DEFAULT_CONFIG allowed to be overriden from DCS
|
||||
config[name] = int(value)
|
||||
return config
|
||||
|
||||
@staticmethod
|
||||
def _build_environment_configuration():
|
||||
ret = defaultdict(dict)
|
||||
|
||||
def _popenv(name):
|
||||
return os.environ.pop(Config.PATRONI_ENV_PREFIX + name.upper(), None)
|
||||
|
||||
for param in ('name', 'namespace', 'scope'):
|
||||
value = _popenv(param)
|
||||
if value:
|
||||
ret[param] = value
|
||||
|
||||
def _set_section_values(section, params):
|
||||
for param in params:
|
||||
value = _popenv(section + '_' + param)
|
||||
if value:
|
||||
ret[section][param] = value
|
||||
|
||||
_set_section_values('restapi', ['listen', 'connect_address', 'certfile', 'keyfile'])
|
||||
_set_section_values('postgresql', ['listen', 'connect_address', 'data_dir', 'pgpass'])
|
||||
|
||||
def _get_auth(name):
|
||||
ret = {}
|
||||
for param in ('username', 'password'):
|
||||
value = _popenv(name + '_' + param)
|
||||
if value:
|
||||
ret[param] = value
|
||||
return ret
|
||||
|
||||
restapi_auth = _get_auth('restapi')
|
||||
if restapi_auth:
|
||||
ret['restapi']['authentication'] = restapi_auth
|
||||
|
||||
authentication = {}
|
||||
for user_type in ('replication', 'superuser'):
|
||||
entry = _get_auth(user_type)
|
||||
if entry:
|
||||
authentication[user_type] = entry
|
||||
|
||||
if authentication:
|
||||
ret['postgresql']['authentication'] = authentication
|
||||
|
||||
users = {}
|
||||
|
||||
def _parse_list(value):
|
||||
if not (value.strip().startswith('-') or '[' in value):
|
||||
value = '[{0}]'.format(value)
|
||||
try:
|
||||
return yaml.safe_load(value)
|
||||
except Exception:
|
||||
logger.exception('Exception when parsing list %s', value)
|
||||
return None
|
||||
|
||||
for param in list(os.environ.keys()):
|
||||
if param.startswith(Config.PATRONI_ENV_PREFIX):
|
||||
name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
|
||||
if name and suffix:
|
||||
# PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT)
|
||||
if suffix in ('HOST', 'HOSTS', 'PORT') and '_' not in name:
|
||||
value = os.environ.pop(param)
|
||||
if suffix == 'PORT':
|
||||
value = value and parse_int(value)
|
||||
elif suffix == 'HOSTS':
|
||||
value = value and _parse_list(value)
|
||||
if value:
|
||||
ret[name.lower()][suffix.lower()] = value
|
||||
# PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>
|
||||
# CREATE USER "<username>" WITH <OPTIONS> PASSWORD '<password>'
|
||||
elif suffix == 'PASSWORD':
|
||||
password = os.environ.pop(param)
|
||||
if password:
|
||||
users[name] = {'password': password}
|
||||
options = os.environ.pop(param[:-9] + '_OPTIONS', None)
|
||||
options = options and _parse_list(options)
|
||||
if options:
|
||||
users[name]['options'] = options
|
||||
if users:
|
||||
ret['bootstrap']['users'] = users
|
||||
|
||||
return ret
|
||||
|
||||
def _build_effective_configuration(self, dynamic_configuration, local_configuration):
|
||||
config = self._safe_copy_dynamic_configuration(dynamic_configuration)
|
||||
for name, value in local_configuration.items():
|
||||
if name == 'postgresql':
|
||||
for name, value in (value or {}).items():
|
||||
if name == 'parameters':
|
||||
config['postgresql'][name].update(self._process_postgresql_parameters(value, True))
|
||||
elif name != 'use_slots': # replication slots must be enabled/disabled globally
|
||||
config['postgresql'][name] = deepcopy(value)
|
||||
elif name not in config:
|
||||
config[name] = deepcopy(value) if value else {}
|
||||
|
||||
# restapi server expects to get restapi.auth = 'username:password'
|
||||
if 'authentication' in config['restapi']:
|
||||
restapi = config['restapi']
|
||||
auth = restapi['authentication']
|
||||
restapi['auth'] = '{0}:{1}'.format(auth['username'], auth['password'])
|
||||
|
||||
# special treatment for old config
|
||||
|
||||
# 'exhibitor' inside 'zookeeper':
|
||||
if 'zookeeper' in config and 'exhibitor' in config['zookeeper']:
|
||||
config['exhibitor'] = config['zookeeper'].pop('exhibitor')
|
||||
config.pop('zookeeper')
|
||||
|
||||
pg_config = config['postgresql']
|
||||
# no 'authentication' in 'postgresql', but 'replication' and 'superuser'
|
||||
if 'authentication' not in pg_config:
|
||||
pg_config['use_pg_rewind'] = 'pg_rewind' in pg_config
|
||||
pg_config['authentication'] = {u: pg_config[u] for u in ('replication', 'superuser') if u in pg_config}
|
||||
# no 'superuser' in 'postgresql'.'authentication'
|
||||
if 'superuser' not in pg_config['authentication'] and 'pg_rewind' in pg_config:
|
||||
pg_config['authentication']['superuser'] = pg_config['pg_rewind']
|
||||
|
||||
# no 'name' in config
|
||||
if 'name' not in config and 'name' in pg_config:
|
||||
config['name'] = pg_config['name']
|
||||
|
||||
pg_config.update({p: config[p] for p in ('name', 'scope', 'retry_timeout',
|
||||
'maximum_lag_on_failover') if p in config})
|
||||
|
||||
return config
|
||||
|
||||
def get(self, key, default=None):
|
||||
return self.__effective_configuration.get(key, default)
|
||||
|
||||
def __contains__(self, key):
|
||||
return key in self.__effective_configuration
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.__effective_configuration[key]
|
||||
|
||||
def copy(self):
|
||||
return deepcopy(self.__effective_configuration)
|
||||
+160
-192
@@ -2,94 +2,87 @@
|
||||
Patroni Control
|
||||
'''
|
||||
|
||||
import base64
|
||||
import click
|
||||
import datetime
|
||||
import dateutil.parser
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import yaml
|
||||
import json
|
||||
import time
|
||||
import psycopg2
|
||||
import random
|
||||
import requests
|
||||
import sys
|
||||
import time
|
||||
import tzlocal
|
||||
import yaml
|
||||
|
||||
from click import ClickException
|
||||
from patroni.config import Config
|
||||
from patroni.dcs import get_dcs as _get_dcs
|
||||
from patroni.exceptions import PatroniException
|
||||
from patroni.postgresql import get_conn_kwargs
|
||||
import datetime
|
||||
from prettytable import PrettyTable
|
||||
from six.moves.urllib_parse import urlparse
|
||||
import logging
|
||||
|
||||
from .etcd import Etcd
|
||||
from .exceptions import PatroniCtlException
|
||||
from .postgresql import parseurl
|
||||
|
||||
CONFIG_DIR_PATH = click.get_app_dir('patroni')
|
||||
CONFIG_FILE_PATH = os.path.join(CONFIG_DIR_PATH, 'patronictl.yaml')
|
||||
LOGLEVEL = 'WARNING'
|
||||
DCS_DEFAULTS = {'zookeeper': {'port': 2181, 'template': "zookeeper:\n hosts: ['{host}:{port}']"},
|
||||
'exhibitor': {'port': 8181, 'template': "exhibitor:\n hosts: [{host}]\n port: {port}"},
|
||||
'consul': {'port': 8500, 'template': "consul:\n host: '{host}:{port}'"},
|
||||
'etcd': {'port': 4001, 'template': "etcd:\n host: '{host}:{port}'"}}
|
||||
|
||||
|
||||
class PatroniCtlException(ClickException):
|
||||
pass
|
||||
|
||||
|
||||
def parse_dcs(dcs):
|
||||
if dcs is None:
|
||||
return None
|
||||
"""
|
||||
Break up the provided dcs string
|
||||
>>> parse_dcs('localhost') == {'scheme': 'etcd', 'hostname': 'localhost', 'port': 4001}
|
||||
True
|
||||
>>> parse_dcs('localhost:8500') == {'scheme': 'consul', 'hostname': 'localhost', 'port': 8500}
|
||||
True
|
||||
>>> parse_dcs('zookeeper://localhost') == {'scheme': 'zookeeper', 'hostname': 'localhost', 'port': 2181}
|
||||
True
|
||||
"""
|
||||
|
||||
if not dcs:
|
||||
return {}
|
||||
|
||||
parsed = urlparse(dcs)
|
||||
scheme = parsed.scheme
|
||||
if scheme == '' and parsed.netloc == '':
|
||||
parsed = urlparse('//' + dcs)
|
||||
port = int(parsed.port) if parsed.port else None
|
||||
|
||||
if scheme == '':
|
||||
scheme = ([k for k, v in DCS_DEFAULTS.items() if v['port'] == port] or ['etcd'])[0]
|
||||
elif scheme not in DCS_DEFAULTS:
|
||||
raise PatroniCtlException('Unknown dcs scheme: {}'.format(scheme))
|
||||
default_schemes = {'2181': 'zookeeper', '8500': 'consul'}
|
||||
scheme = default_schemes.get(str(parsed.port), 'etcd')
|
||||
|
||||
dcs_info = DCS_DEFAULTS[scheme]
|
||||
return yaml.load(dcs_info['template'].format(host=parsed.hostname or 'localhost', port=port or dcs_info['port']))
|
||||
port = parsed.port
|
||||
if port is None:
|
||||
default_ports = {'consul': 8500, 'zookeeper': 2181}
|
||||
port = default_ports.get(str(scheme), 4001)
|
||||
|
||||
return {'scheme': str(scheme), 'hostname': str(parsed.hostname), 'port': int(port)}
|
||||
|
||||
|
||||
def load_config(path, dcs):
|
||||
logging.debug('Loading configuration from file %s', path)
|
||||
config = {}
|
||||
old_argv = list(sys.argv)
|
||||
logging.debug('Loading configuration from file {}'.format(path))
|
||||
config = dict()
|
||||
try:
|
||||
sys.argv[1] = path
|
||||
if Config.PATRONI_CONFIG_VARIABLE not in os.environ:
|
||||
for p in ('PATRONI_RESTAPI_LISTEN', 'PATRONI_POSTGRESQL_DATA_DIR'):
|
||||
if p not in os.environ:
|
||||
os.environ[p] = '.'
|
||||
config = Config().copy()
|
||||
finally:
|
||||
sys.argv = old_argv
|
||||
with open(path, 'rb') as fd:
|
||||
config = yaml.safe_load(fd)
|
||||
except:
|
||||
logging.exception('Could not load configuration file')
|
||||
|
||||
dcs = parse_dcs(dcs) or parse_dcs(config.get('dcs_api')) or {}
|
||||
if dcs:
|
||||
for d in DCS_DEFAULTS:
|
||||
config.pop(d, None)
|
||||
config.update(dcs)
|
||||
config['dcs'] = parse_dcs(dcs)
|
||||
else:
|
||||
config['dcs'] = parse_dcs(config.get('dcs_api'))
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def store_config(config, path):
|
||||
dir_path = os.path.dirname(path)
|
||||
if dir_path and not os.path.isdir(dir_path):
|
||||
os.makedirs(dir_path)
|
||||
if dir_path:
|
||||
if not os.path.isdir(dir_path):
|
||||
os.makedirs(dir_path)
|
||||
with open(path, 'w') as fd:
|
||||
yaml.dump(config, fd)
|
||||
|
||||
|
||||
option_config_file = click.option('--config-file', '-c', help='Configuration file', default=CONFIG_FILE_PATH)
|
||||
option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, json)', default='pretty')
|
||||
option_format = click.option('--format', '-f', help='Output format (pretty, json)', default='pretty')
|
||||
option_dcs = click.option('--dcs', '-d', help='Use this DCS', envvar='DCS')
|
||||
option_watchrefresh = click.option('-w', '--watch', type=float, help='Auto update the screen every X seconds')
|
||||
option_watch = click.option('-W', is_flag=True, help='Auto update the screen every 2 seconds')
|
||||
@@ -106,33 +99,23 @@ def ctl(ctx):
|
||||
|
||||
|
||||
def get_dcs(config, scope):
|
||||
config.setdefault('scope', scope)
|
||||
config.setdefault('name', scope)
|
||||
try:
|
||||
return _get_dcs(config)
|
||||
except PatroniException as e:
|
||||
raise PatroniCtlException(str(e))
|
||||
scheme, hostname, port = map(config.get('dcs', {}).get, ('scheme', 'hostname', 'port'))
|
||||
|
||||
if scheme == 'etcd':
|
||||
return Etcd(name=scope, config={'scope': scope, 'host': '{}:{}'.format(hostname, port)})
|
||||
|
||||
raise PatroniCtlException('Can not find suitable configuration of distributed configuration store')
|
||||
|
||||
|
||||
def auth_header(config):
|
||||
if config.get('restapi', {}).get('auth', ''):
|
||||
return {'Authorization': 'Basic ' + base64.b64encode(config['restapi']['auth'].encode('utf-8')).decode('utf-8')}
|
||||
|
||||
|
||||
def post_patroni(member, endpoint, content, headers=None):
|
||||
headers = headers or {}
|
||||
def post_patroni(member, endpoint, content, headers={'Content-Type': 'application/json'}):
|
||||
url = urlparse(member.api_url)
|
||||
logging.debug(url)
|
||||
if 'Content-Type' not in headers:
|
||||
headers['Content-Type'] = 'application/json'
|
||||
return requests.post('{0}://{1}/{2}'.format(url.scheme, url.netloc, endpoint),
|
||||
headers=headers,
|
||||
return requests.post('{}://{}/{}'.format(url.scheme, url.netloc, endpoint), headers=headers,
|
||||
data=json.dumps(content), timeout=60)
|
||||
|
||||
|
||||
def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True, delimiter='\t'):
|
||||
rows = rows or []
|
||||
if fmt == 'pretty':
|
||||
def print_output(columns, rows=[], alignment=None, format='pretty', header=True, delimiter='\t'):
|
||||
if format == 'pretty':
|
||||
t = PrettyTable(columns)
|
||||
for k, v in (alignment or {}).items():
|
||||
t.align[k] = v
|
||||
@@ -141,15 +124,18 @@ def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True,
|
||||
click.echo(t)
|
||||
return
|
||||
|
||||
if fmt == 'json':
|
||||
elements = [dict(zip(columns, r)) for r in rows]
|
||||
if format == 'json':
|
||||
elements = list()
|
||||
for r in rows:
|
||||
elements.append(dict(zip(columns, r)))
|
||||
|
||||
click.echo(json.dumps(elements))
|
||||
|
||||
if fmt == 'tsv':
|
||||
if format == 'tsv':
|
||||
if columns is not None and header:
|
||||
click.echo(delimiter.join(columns) + '\n')
|
||||
|
||||
for r in rows:
|
||||
for r in rows or []:
|
||||
c = [str(c) for c in r]
|
||||
click.echo(delimiter.join(c))
|
||||
|
||||
@@ -182,13 +168,14 @@ def watching(w, watch, max_count=None, clear=True):
|
||||
yield 0
|
||||
|
||||
|
||||
def build_connect_parameters(conn_url, connect_parameters):
|
||||
params = get_conn_kwargs(conn_url, connect_parameters)
|
||||
params.update({'fallback_application_name': 'Patroni ctl', 'connect_timeout': '5'})
|
||||
if 'database' in connect_parameters:
|
||||
params['database'] = connect_parameters['database']
|
||||
else:
|
||||
params.pop('database')
|
||||
def build_connect_parameters(conn_url, connect_parameters={}):
|
||||
params = connect_parameters.copy()
|
||||
parsed = parseurl(conn_url)
|
||||
params['host'] = parsed['host']
|
||||
params['port'] = parsed['port']
|
||||
params['fallback_application_name'] = 'Patroni ctl'
|
||||
params['connect_timeout'] = '5'
|
||||
|
||||
return params
|
||||
|
||||
|
||||
@@ -205,18 +192,20 @@ def get_all_members(cluster, role='master'):
|
||||
|
||||
|
||||
def get_any_member(cluster, role='master', member=None):
|
||||
members = get_all_members(cluster, role)
|
||||
members = get_all_members(cluster=cluster, role=role)
|
||||
for m in members:
|
||||
if member is None or m.name == member:
|
||||
return m
|
||||
|
||||
return None
|
||||
|
||||
def get_cursor(cluster, connect_parameters, role='master', member=None):
|
||||
member = get_any_member(cluster, role=role, member=member)
|
||||
|
||||
def get_cursor(cluster, role='master', member=None, connect_parameters={}):
|
||||
member = get_any_member(cluster=cluster, role=role, member=member)
|
||||
if member is None:
|
||||
return None
|
||||
|
||||
params = build_connect_parameters(member.conn_url, connect_parameters)
|
||||
params = build_connect_parameters(member.conn_url, connect_parameters=connect_parameters)
|
||||
|
||||
conn = psycopg2.connect(**params)
|
||||
conn.autocommit = True
|
||||
@@ -248,23 +237,21 @@ def dsn(cluster_name, config_file, dcs, role, member):
|
||||
if member is None and role is None:
|
||||
role = 'master'
|
||||
|
||||
_, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
|
||||
m = get_any_member(cluster, role=role, member=member)
|
||||
config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
|
||||
m = get_any_member(cluster=cluster, role=role, member=member)
|
||||
if m is None:
|
||||
raise PatroniCtlException('Can not find a suitable member')
|
||||
|
||||
params = get_conn_kwargs(m.conn_url)
|
||||
click.echo('host={host} port={port}'.format(**params))
|
||||
params = build_connect_parameters(m.conn_url)
|
||||
click.echo('host={} port={}'.format(params['host'], params['port']))
|
||||
|
||||
|
||||
@ctl.command('query', help='Query a Patroni PostgreSQL member')
|
||||
@click.argument('cluster_name')
|
||||
@option_config_file
|
||||
@option_format
|
||||
@click.option('--format', 'fmt', help='Output format (pretty, json)', default='tsv')
|
||||
@click.option('--file', '-f', 'p_file', help='Execute the SQL commands from this file', type=click.File('rb'))
|
||||
@click.option('--password', help='force password prompt', is_flag=True)
|
||||
@click.option('-U', '--username', help='database user name', type=str)
|
||||
@click.option('--format', help='Output format (pretty, json)', default='tsv')
|
||||
@click.option('--file', '-f', help='Execute the SQL commands from this file', type=click.File('rb'))
|
||||
@option_dcs
|
||||
@option_watch
|
||||
@option_watchrefresh
|
||||
@@ -273,7 +260,6 @@ def dsn(cluster_name, config_file, dcs, role, member):
|
||||
@click.option('--member', '-m', help='Query a specific member', type=str)
|
||||
@click.option('--delimiter', help='The column delimiter', default='\t')
|
||||
@click.option('--command', '-c', help='The SQL commands to execute')
|
||||
@click.option('-d', '--dbname', help='database name to connect to', type=str)
|
||||
def query(
|
||||
cluster_name,
|
||||
config_file,
|
||||
@@ -284,56 +270,42 @@ def query(
|
||||
watch,
|
||||
delimiter,
|
||||
command,
|
||||
p_file,
|
||||
password,
|
||||
username,
|
||||
dbname,
|
||||
fmt='tsv',
|
||||
file,
|
||||
format='tsv',
|
||||
):
|
||||
if role is not None and member is not None:
|
||||
raise PatroniCtlException('--role and --member are mutually exclusive options')
|
||||
if member is None and role is None:
|
||||
role = 'master'
|
||||
|
||||
if p_file is not None and command is not None:
|
||||
if file is not None and command is not None:
|
||||
raise PatroniCtlException('--file and --command are mutually exclusive options')
|
||||
|
||||
if p_file is None and command is None:
|
||||
raise PatroniCtlException('You need to specify either --command or --file')
|
||||
if file is not None:
|
||||
command = file.read()
|
||||
|
||||
connect_parameters = dict()
|
||||
if username:
|
||||
connect_parameters['username'] = username
|
||||
if password:
|
||||
connect_parameters['password'] = click.prompt('Password', hide_input=True, type=str)
|
||||
if dbname:
|
||||
connect_parameters['database'] = dbname
|
||||
|
||||
if p_file is not None:
|
||||
command = p_file.read()
|
||||
|
||||
_, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
|
||||
config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
|
||||
|
||||
cursor = None
|
||||
for _ in watching(w, watch, clear=False):
|
||||
|
||||
output, cursor = query_member(cluster, cursor, member, role, command, connect_parameters)
|
||||
print_output(None, output, fmt=fmt, delimiter=delimiter)
|
||||
output, cursor = query_member(cluster=cluster, cursor=cursor, member=member, role=role, command=command)
|
||||
print_output(None, output, format=format, delimiter=delimiter)
|
||||
|
||||
if cursor is None:
|
||||
cluster = dcs.get_cluster()
|
||||
|
||||
|
||||
def query_member(cluster, cursor, member, role, command, connect_parameters):
|
||||
def query_member(cluster, cursor, member, role, command):
|
||||
try:
|
||||
if cursor is None:
|
||||
cursor = get_cursor(cluster, connect_parameters, role=role, member=member)
|
||||
cursor = get_cursor(cluster, role=role, member=member)
|
||||
|
||||
if cursor is None:
|
||||
if role is None:
|
||||
message = 'No connection to member {0} is available'.format(member)
|
||||
message = 'No connection to member {} is available'.format(member)
|
||||
else:
|
||||
message = 'No connection to role={0} is available'.format(role)
|
||||
message = 'No connection to role={} is available'.format(role)
|
||||
logging.debug(message)
|
||||
return [[timestamp(0), message]], None
|
||||
|
||||
@@ -352,7 +324,7 @@ def query_member(cluster, cursor, member, role, command, connect_parameters):
|
||||
cursor.connection.close()
|
||||
message = oe.pgcode or oe.pgerror or str(oe)
|
||||
message = message.replace('\n', ' ')
|
||||
return [[timestamp(0), 'ERROR, SQLSTATE: {0}'.format(message)]], None
|
||||
return [[timestamp(0), 'ERROR, SQLSTATE: {}'.format(message)]], None
|
||||
|
||||
|
||||
@ctl.command('remove', help='Remove cluster from DCS')
|
||||
@@ -360,10 +332,13 @@ def query_member(cluster, cursor, member, role, command, connect_parameters):
|
||||
@option_config_file
|
||||
@option_format
|
||||
@option_dcs
|
||||
def remove(config_file, cluster_name, fmt, dcs):
|
||||
_, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
|
||||
def remove(config_file, cluster_name, format, dcs):
|
||||
config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
|
||||
|
||||
output_members(cluster, cluster_name, fmt)
|
||||
if not isinstance(dcs, Etcd):
|
||||
raise PatroniCtlException('We have not implemented this for DCS of type {}'.format(type(dcs)))
|
||||
|
||||
output_members(cluster, format=format)
|
||||
|
||||
confirm = click.prompt('Please confirm the cluster name to remove', type=str)
|
||||
if confirm != cluster_name:
|
||||
@@ -371,17 +346,17 @@ def remove(config_file, cluster_name, fmt, dcs):
|
||||
|
||||
message = 'Yes I am aware'
|
||||
confirm = \
|
||||
click.prompt('You are about to remove all information in DCS for {0}, please type: "{1}"'.format(cluster_name,
|
||||
click.prompt('You are about to remove all information in DCS for {}, please type: "{}"'.format(cluster_name,
|
||||
message), type=str)
|
||||
if message != confirm:
|
||||
raise PatroniCtlException('You did not exactly type "{0}"'.format(message))
|
||||
raise PatroniCtlException('You did not exactly type "{}"'.format(message))
|
||||
|
||||
if cluster.leader:
|
||||
confirm = click.prompt('This cluster currently is healthy. Please specify the master name to continue')
|
||||
if confirm != cluster.leader.name:
|
||||
raise PatroniCtlException('You did not specify the current master of the cluster')
|
||||
|
||||
dcs.delete_cluster()
|
||||
dcs.client.delete(dcs._base_path, recursive=True)
|
||||
|
||||
|
||||
def wait_for_leader(dcs, timeout=30):
|
||||
@@ -398,28 +373,30 @@ def wait_for_leader(dcs, timeout=30):
|
||||
raise PatroniCtlException('Timeout occured')
|
||||
|
||||
|
||||
def empty_post_to_members(cluster, member_names, force, endpoint, headers=None):
|
||||
candidates = {m.name: m for m in cluster.members}
|
||||
def empty_post_to_members(cluster, member_names, force, endpoint):
|
||||
candidates = dict()
|
||||
for m in cluster.members:
|
||||
candidates[m.name] = m
|
||||
|
||||
if not member_names:
|
||||
member_names = [click.prompt('Which member do you want to {0} [{1}]?'.format(endpoint,
|
||||
if len(member_names) == 0:
|
||||
member_names = [click.prompt('Which member do you want to {} [{}]?'.format(endpoint,
|
||||
', '.join(candidates.keys())), type=str, default='')]
|
||||
|
||||
for mn in member_names:
|
||||
if mn not in candidates:
|
||||
raise PatroniCtlException('{0} is not a member of cluster'.format(mn))
|
||||
if mn not in candidates.keys():
|
||||
raise PatroniCtlException('{} is not a member of cluster'.format(mn))
|
||||
|
||||
if not force:
|
||||
confirm = click.confirm('Are you sure you want to {0} members {1}?'.format(endpoint, ', '.join(member_names)))
|
||||
confirm = click.confirm('Are you sure you want to {} members {}?'.format(endpoint, ', '.join(member_names)))
|
||||
if not confirm:
|
||||
raise PatroniCtlException('Aborted {0}'.format(endpoint))
|
||||
raise PatroniCtlException('Aborted {}'.format(endpoint))
|
||||
|
||||
for mn in member_names:
|
||||
r = post_patroni(candidates[mn], endpoint, '', headers)
|
||||
r = post_patroni(candidates[mn], endpoint, '')
|
||||
if r.status_code != 200:
|
||||
click.echo('{0} failed for member {1}, status code={2}, ({3})'.format(endpoint, mn, r.status_code, r.text))
|
||||
click.echo('{} failed for member {}, status code={}, ({})'.format(endpoint, mn, r.status_code, r.text))
|
||||
else:
|
||||
click.echo('Succesful {0} on member {1}'.format(endpoint, mn))
|
||||
click.echo('Succesful {} on member {}'.format(endpoint, mn))
|
||||
|
||||
|
||||
def ctl_load_config(cluster_name, config_file, dcs):
|
||||
@@ -435,26 +412,26 @@ def ctl_load_config(cluster_name, config_file, dcs):
|
||||
@click.argument('member_names', nargs=-1)
|
||||
@click.option('--role', '-r', help='Restart only members with this role', default='any',
|
||||
type=click.Choice(['master', 'replica', 'any']))
|
||||
@click.option('--any', 'p_any', help='Restart a single member only', is_flag=True)
|
||||
@click.option('--any', help='Restart a single member only', is_flag=True)
|
||||
@option_config_file
|
||||
@option_force
|
||||
@option_dcs
|
||||
def restart(cluster_name, member_names, config_file, dcs, force, role, p_any):
|
||||
def restart(cluster_name, member_names, config_file, dcs, force, role, any):
|
||||
config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
|
||||
|
||||
role_names = [m.name for m in get_all_members(cluster, role)]
|
||||
role_names = [m.name for m in get_all_members(cluster=cluster, role=role)]
|
||||
|
||||
if member_names:
|
||||
if len(member_names) > 0:
|
||||
member_names = list(set(member_names) & set(role_names))
|
||||
else:
|
||||
member_names = role_names
|
||||
|
||||
if p_any:
|
||||
if any:
|
||||
random.shuffle(member_names)
|
||||
member_names = member_names[:1]
|
||||
|
||||
output_members(cluster, cluster_name)
|
||||
empty_post_to_members(cluster, member_names, force, 'restart', auth_header(config))
|
||||
output_members(cluster)
|
||||
empty_post_to_members(cluster, member_names, force, 'restart')
|
||||
|
||||
|
||||
@ctl.command('reinit', help='Reinitialize cluster member')
|
||||
@@ -465,19 +442,17 @@ def restart(cluster_name, member_names, config_file, dcs, force, role, p_any):
|
||||
@option_dcs
|
||||
def reinit(cluster_name, member_names, config_file, dcs, force):
|
||||
config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
|
||||
empty_post_to_members(cluster, member_names, force, 'reinitialize', auth_header(config))
|
||||
empty_post_to_members(cluster, member_names, force, 'reinitialize')
|
||||
|
||||
|
||||
@ctl.command('failover', help='Failover to a replica')
|
||||
@click.argument('cluster_name')
|
||||
@click.option('--master', help='The name of the current master', default=None)
|
||||
@click.option('--candidate', help='The name of the candidate', default=None)
|
||||
@click.option('--scheduled', help='Timestamp of a scheduled failover in unambiguous format (e.g. ISO 8601)',
|
||||
default=None)
|
||||
@click.option('--force', is_flag=True)
|
||||
@option_config_file
|
||||
@option_dcs
|
||||
def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled):
|
||||
def failover(config_file, cluster_name, master, candidate, force, dcs):
|
||||
"""
|
||||
We want to trigger a failover for the specified cluster name.
|
||||
|
||||
@@ -497,13 +472,13 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
|
||||
master = click.prompt('Master', type=str, default=cluster.leader.member.name)
|
||||
|
||||
if cluster.leader.member.name != master:
|
||||
raise PatroniCtlException('Member {0} is not the leader of cluster {1}'.format(master, cluster_name))
|
||||
raise PatroniCtlException('Member {} is not the leader of cluster {}'.format(master, cluster_name))
|
||||
|
||||
candidate_names = [str(m.name) for m in cluster.members if m.name != master]
|
||||
# We sort the names for consistent output to the client
|
||||
candidate_names.sort()
|
||||
|
||||
if not candidate_names:
|
||||
if len(candidate_names) == 0:
|
||||
raise PatroniCtlException('No candidates found to failover to')
|
||||
|
||||
if candidate is None and not force:
|
||||
@@ -513,60 +488,52 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
|
||||
raise PatroniCtlException('Failover target and source are the same.')
|
||||
|
||||
if candidate and candidate not in candidate_names:
|
||||
raise PatroniCtlException('Member {0} does not exist in cluster {1}'.format(candidate, cluster_name))
|
||||
|
||||
if scheduled is None and not force:
|
||||
scheduled = click.prompt('When should the failover take place (e.g. 2015-10-01T14:30) ', type=str,
|
||||
default='now')
|
||||
|
||||
if (scheduled or 'now') == 'now':
|
||||
scheduled_at = None
|
||||
else:
|
||||
try:
|
||||
scheduled_at = dateutil.parser.parse(scheduled)
|
||||
if scheduled_at.tzinfo is None:
|
||||
scheduled_at = tzlocal.get_localzone().localize(scheduled_at)
|
||||
except (ValueError, TypeError):
|
||||
message = 'Unable to parse scheduled timestamp ({0}). It should be in an unambiguous format (e.g. ISO 8601)'
|
||||
raise PatroniCtlException(message.format(scheduled))
|
||||
scheduled_at = scheduled_at.isoformat()
|
||||
|
||||
failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at}
|
||||
logging.debug(failover_value)
|
||||
raise PatroniCtlException('Member {} does not exist in cluster {}'.format(candidate, cluster_name))
|
||||
|
||||
# By now we have established that the leader exists and the candidate exists
|
||||
click.echo('Current cluster topology')
|
||||
output_members(dcs.get_cluster(), cluster_name)
|
||||
output_members(dcs.get_cluster(), name=cluster_name)
|
||||
|
||||
if not force:
|
||||
a = \
|
||||
click.confirm('Are you sure you want to failover cluster {0}, demoting current master {1}?'.format(
|
||||
click.confirm('Are you sure you want to failover cluster {}, demoting current master {}?'.format(
|
||||
cluster_name, master))
|
||||
if not a:
|
||||
raise PatroniCtlException('Aborting failover')
|
||||
|
||||
failover_value = '{}:{}'.format(master, candidate or '')
|
||||
|
||||
t_started = time.time()
|
||||
r = None
|
||||
try:
|
||||
r = post_patroni(cluster.leader.member, 'failover', failover_value, auth_header(config))
|
||||
if r.status_code in (200, 202):
|
||||
r = post_patroni(cluster.leader.member, 'failover', {'leader': master, 'member': candidate or ''})
|
||||
if r.status_code == 200:
|
||||
logging.debug(r)
|
||||
logging.debug(r.text)
|
||||
cluster = dcs.get_cluster()
|
||||
logging.debug(cluster)
|
||||
click.echo('{0} {1}'.format(timestamp(), r.text))
|
||||
click.echo(timestamp() + ' Failing over to new leader: {}'.format(cluster.leader.member.name))
|
||||
else:
|
||||
click.echo('Failover failed, details: {0}, {1}'.format(r.status_code, r.text))
|
||||
click.echo('Failover failed, details: {}, {}'.format(r.status_code, r.text))
|
||||
return
|
||||
except Exception:
|
||||
except:
|
||||
logging.exception(r)
|
||||
logging.warning('Failing over to DCS')
|
||||
click.echo(timestamp() + ' Could not failover using Patroni api, falling back to DCS')
|
||||
click.echo(timestamp() + ' Initializing failover from master {0}'.format(master))
|
||||
dcs.manual_failover(master, candidate, scheduled_at=failover_value)
|
||||
dcs.set_failover_value(failover_value)
|
||||
click.echo(timestamp() + ' Initialized failover from master {}'.format(master))
|
||||
# The failover process should within a minute update the failover key, we will keep watching it until it changes
|
||||
# or we timeout
|
||||
cluster = wait_for_leader(dcs, timeout=60)
|
||||
if cluster.leader.member.name == master:
|
||||
click.echo('Failover failed, master did not change after {:0.1f} seconds'.format(time.time() - t_started))
|
||||
return
|
||||
|
||||
output_members(cluster, cluster_name)
|
||||
click.echo(timestamp() + ' Failover completed in {:0.1f} seconds, new leader is {}'.format(time.time() - t_started,
|
||||
str(cluster.leader.member.name)))
|
||||
output_members(cluster, name=cluster_name)
|
||||
|
||||
|
||||
def output_members(cluster, name, fmt='pretty'):
|
||||
def output_members(cluster, name=None, format='pretty'):
|
||||
rows = []
|
||||
logging.debug(cluster)
|
||||
leader_name = None
|
||||
@@ -584,11 +551,12 @@ def output_members(cluster, name, fmt='pretty'):
|
||||
if m.name == leader_name:
|
||||
leader = '*'
|
||||
|
||||
host = get_conn_kwargs(m.conn_url)['host']
|
||||
host = build_connect_parameters(m.conn_url)['host']
|
||||
|
||||
xlog_location = m.data.get('xlog_location') or 0
|
||||
lag = ''
|
||||
if (xlog_location_cluster >= xlog_location):
|
||||
xlog_location = m.data.get('xlog_location')
|
||||
if xlog_location is None or (xlog_location_cluster < xlog_location):
|
||||
lag = ''
|
||||
else:
|
||||
lag = round((xlog_location_cluster - xlog_location)/1024/1024)
|
||||
|
||||
rows.append([
|
||||
@@ -610,7 +578,7 @@ def output_members(cluster, name, fmt='pretty'):
|
||||
]
|
||||
alignment = {'Cluster': 'l', 'Member': 'l', 'Host': 'l', 'Lag in MB': 'r'}
|
||||
|
||||
print_output(columns, rows, alignment, fmt)
|
||||
print_output(columns, rows, alignment, format)
|
||||
|
||||
|
||||
@ctl.command('list', help='List the Patroni members for a given Patroni')
|
||||
@@ -620,17 +588,17 @@ def output_members(cluster, name, fmt='pretty'):
|
||||
@option_watch
|
||||
@option_watchrefresh
|
||||
@option_dcs
|
||||
def members(config_file, cluster_names, fmt, watch, w, dcs):
|
||||
if not cluster_names:
|
||||
def members(config_file, cluster_names, format, watch, w, dcs):
|
||||
if len(cluster_names) == 0:
|
||||
logging.warning('Listing members: No cluster names were provided')
|
||||
return
|
||||
|
||||
config = load_config(config_file, dcs)
|
||||
for cluster_name in cluster_names:
|
||||
dcs = get_dcs(config, cluster_name)
|
||||
for cn in cluster_names:
|
||||
dcs = get_dcs(config, cn)
|
||||
|
||||
for _ in watching(w, watch):
|
||||
output_members(dcs.get_cluster(), cluster_name, fmt)
|
||||
output_members(dcs.get_cluster(), name=cn, format=format)
|
||||
|
||||
|
||||
def timestamp(precision=6):
|
||||
|
||||
@@ -1,14 +1,8 @@
|
||||
import abc
|
||||
import dateutil
|
||||
import importlib
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
import six
|
||||
|
||||
from collections import namedtuple
|
||||
from patroni.exceptions import PatroniException
|
||||
from random import randint
|
||||
from patroni.exceptions import DCSError
|
||||
from six.moves.urllib_parse import urlparse, urlunparse, parse_qsl
|
||||
from threading import Event, Lock
|
||||
|
||||
@@ -30,27 +24,6 @@ def parse_connection_string(value):
|
||||
return conn_url, api_url
|
||||
|
||||
|
||||
def get_dcs(config):
|
||||
available_implementations = set()
|
||||
for module in os.listdir(os.path.dirname(__file__)):
|
||||
if module.endswith('.py') and not module.startswith('__'): # find module
|
||||
module_name = module[:-3].lower()
|
||||
module = importlib.import_module(__package__ + '.' + module[:-3])
|
||||
for name in filter(lambda name: not name.startswith('__'), dir(module)): # iterate through module content
|
||||
value = getattr(module, name)
|
||||
name = name.lower()
|
||||
# try to find implementation of AbstractDCS interface, class name must match with module_name
|
||||
if inspect.isclass(value) and issubclass(value, AbstractDCS) and name == module_name:
|
||||
available_implementations.add(name)
|
||||
if name in config: # which has configuration section in the config file
|
||||
# propagate some parameters
|
||||
config[name].update({p: config[p] for p in ('namespace', 'name',
|
||||
'scope', 'ttl', 'retry_timeout') if p in config})
|
||||
return value(config[name])
|
||||
raise PatroniException("""Can not find suitable configuration of distributed configuration store
|
||||
Available implementations: """ + ', '.join(available_implementations))
|
||||
|
||||
|
||||
class Member(namedtuple('Member', 'index,name,session,data')):
|
||||
|
||||
"""Immutable object (namedtuple) which represents single member of PostgreSQL cluster.
|
||||
@@ -78,33 +51,21 @@ class Member(namedtuple('Member', 'index,name,session,data')):
|
||||
else:
|
||||
try:
|
||||
data = json.loads(data)
|
||||
except (TypeError, ValueError):
|
||||
except:
|
||||
data = {}
|
||||
return Member(index, name, session, data)
|
||||
|
||||
@property
|
||||
def conn_url(self):
|
||||
return self.data.get('conn_url')
|
||||
return self.data.get('conn_url', None)
|
||||
|
||||
@property
|
||||
def api_url(self):
|
||||
return self.data.get('api_url')
|
||||
|
||||
@property
|
||||
def tags(self):
|
||||
return self.data.get('tags', {})
|
||||
return self.data.get('api_url', None)
|
||||
|
||||
@property
|
||||
def nofailover(self):
|
||||
return self.tags.get('nofailover', False)
|
||||
|
||||
@property
|
||||
def replicatefrom(self):
|
||||
return self.tags.get('replicatefrom')
|
||||
|
||||
@property
|
||||
def clonefrom(self):
|
||||
return self.tags.get('clonefrom', False)
|
||||
return self.data.get('tags', {}).get('nofailover', False)
|
||||
|
||||
|
||||
class Leader(namedtuple('Leader', 'index,session,member')):
|
||||
@@ -124,68 +85,19 @@ class Leader(namedtuple('Leader', 'index,session,member')):
|
||||
return self.member.conn_url
|
||||
|
||||
|
||||
class Failover(namedtuple('Failover', 'index,leader,candidate,scheduled_at')):
|
||||
class Failover(namedtuple('Failover', 'index,leader,member')):
|
||||
|
||||
"""
|
||||
>>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader"}'))
|
||||
True
|
||||
>>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader", "member": "cluster_candidate"}'))
|
||||
True
|
||||
>>> Failover.from_node(1, 'null') is None
|
||||
True
|
||||
>>> n = '{"leader": "cluster_leader", "member": "cluster_candidate", "scheduled_at": "2016-01-14T10:09:57.1394Z"}'
|
||||
>>> 'tzinfo=' in str(Failover.from_node(1, n))
|
||||
True
|
||||
>>> Failover.from_node(1, None) is None
|
||||
True
|
||||
>>> Failover.from_node(1, '{}') is None
|
||||
True
|
||||
>>> 'abc' in Failover.from_node(1, 'abc:def')
|
||||
True
|
||||
"""
|
||||
@staticmethod
|
||||
def from_node(index, value):
|
||||
if not value:
|
||||
return None
|
||||
|
||||
try:
|
||||
data = json.loads(value)
|
||||
if not data:
|
||||
return None
|
||||
except ValueError:
|
||||
t = [a.strip() for a in value.split(':')]
|
||||
leader = t[0]
|
||||
candidate = t[1] if len(t) > 1 else None
|
||||
return Failover(index, leader, candidate, None) if leader or candidate else None
|
||||
|
||||
if data.get('scheduled_at'):
|
||||
data['scheduled_at'] = dateutil.parser.parse(data['scheduled_at'])
|
||||
|
||||
return Failover(index, data.get('leader'), data.get('member'), data.get('scheduled_at'))
|
||||
t = [a.strip() for a in value.split(':')] + ['']
|
||||
return Failover(index, t[0], t[1]) if t[0] or t[1] else None
|
||||
|
||||
|
||||
class ClusterConfig(namedtuple('ClusterConfig', 'index,data,modify_index')):
|
||||
|
||||
@staticmethod
|
||||
def from_node(index, data, modify_index=None):
|
||||
"""
|
||||
>>> ClusterConfig.from_node(1, '{') is None
|
||||
True
|
||||
"""
|
||||
|
||||
try:
|
||||
data = json.loads(data)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return ClusterConfig(index, data, modify_index or index)
|
||||
|
||||
|
||||
class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operation,members,failover')):
|
||||
class Cluster(namedtuple('Cluster', 'initialize,leader,last_leader_operation,members,failover')):
|
||||
|
||||
"""Immutable object (namedtuple) which represents PostgreSQL cluster.
|
||||
Consists of the following fields:
|
||||
:param initialize: shows whether this cluster has initialization key stored in DC or not.
|
||||
:param config: global dynamic configuration, reference to `ClusterConfig` object
|
||||
:param initialize: boolean, shows whether this cluster has initialization key stored in DC or not.
|
||||
:param leader: `Leader` object which represents current leader of the cluster
|
||||
:param last_leader_operation: int or long object containing position of last known leader operation.
|
||||
This value is stored in `/optime/leader` key
|
||||
@@ -195,35 +107,26 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operat
|
||||
def is_unlocked(self):
|
||||
return not (self.leader and self.leader.name)
|
||||
|
||||
def has_member(self, member_name):
|
||||
return any(m for m in self.members if m.name == member_name)
|
||||
|
||||
def get_member(self, member_name, fallback_to_leader=True):
|
||||
return ([m for m in self.members if m.name == member_name] or [self.leader if fallback_to_leader else None])[0]
|
||||
class AbstractDCS:
|
||||
|
||||
def get_clone_member(self):
|
||||
candidates = [m for m in self.members if m.clonefrom and (not self.leader or m.name != self.leader.name)]
|
||||
return candidates[randint(0, len(candidates) - 1)] if candidates else self.leader
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
class AbstractDCS(object):
|
||||
__metaclass__ = abc.ABCMeta
|
||||
|
||||
_INITIALIZE = 'initialize'
|
||||
_CONFIG = 'config'
|
||||
_LEADER = 'leader'
|
||||
_FAILOVER = 'failover'
|
||||
_MEMBERS = 'members/'
|
||||
_OPTIME = 'optime'
|
||||
_LEADER_OPTIME = _OPTIME + '/' + _LEADER
|
||||
|
||||
def __init__(self, config):
|
||||
def __init__(self, name, config):
|
||||
"""
|
||||
:param name: name of current instance (the same value as `~Postgresql.name`)
|
||||
:param config: dict, reference to config section of selected DCS.
|
||||
i.e.: `zookeeper` for zookeeper, `etcd` for etcd, etc...
|
||||
"""
|
||||
self._name = config['name']
|
||||
self._namespace = '/{0}'.format(config.get('namespace', '/service/').strip('/'))
|
||||
self._name = name
|
||||
self._namespace = '/{}'.format(config.get('namespace', '/service/').strip('/'))
|
||||
self._base_path = '/'.join([self._namespace, config['scope']])
|
||||
|
||||
self._cluster = None
|
||||
@@ -237,10 +140,6 @@ class AbstractDCS(object):
|
||||
def initialize_path(self):
|
||||
return self.client_path(self._INITIALIZE)
|
||||
|
||||
@property
|
||||
def config_path(self):
|
||||
return self.client_path(self._CONFIG)
|
||||
|
||||
@property
|
||||
def members_path(self):
|
||||
return self.client_path(self._MEMBERS)
|
||||
@@ -261,14 +160,6 @@ class AbstractDCS(object):
|
||||
def leader_optime_path(self):
|
||||
return self.client_path(self._LEADER_OPTIME)
|
||||
|
||||
@abc.abstractmethod
|
||||
def set_ttl(self, ttl):
|
||||
"""Set the new ttl value for leader key"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def set_retry_timeout(self, retry_timeout):
|
||||
"""Set the new value for retry_timeout"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def _load_cluster(self):
|
||||
"""Internally this method should build `Cluster` object which
|
||||
@@ -325,30 +216,23 @@ class AbstractDCS(object):
|
||||
def set_failover_value(self, value, index=None):
|
||||
"""Create or update `/failover` key"""
|
||||
|
||||
def manual_failover(self, leader, candidate, scheduled_at=None, index=None):
|
||||
failover_value = {}
|
||||
if leader:
|
||||
failover_value['leader'] = leader
|
||||
def manual_failover(self, leader, member, index=None):
|
||||
return self.set_failover_value(leader + (':' + member if member else ''), index)
|
||||
|
||||
if candidate:
|
||||
failover_value['member'] = candidate
|
||||
|
||||
if scheduled_at:
|
||||
failover_value['scheduled_at'] = scheduled_at.isoformat()
|
||||
|
||||
return self.set_failover_value(json.dumps(failover_value, separators=(',', ':')), index)
|
||||
def current_leader(self):
|
||||
try:
|
||||
cluster = self.get_cluster()
|
||||
return None if cluster.is_unlocked() else cluster.leader
|
||||
except DCSError:
|
||||
return None
|
||||
|
||||
@abc.abstractmethod
|
||||
def set_config_value(self, value, index=None):
|
||||
"""Create or update `/config` key"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def touch_member(self, data, ttl=None):
|
||||
def touch_member(self, connection_string, ttl=None):
|
||||
"""Update member key in DCS.
|
||||
This method should create or update key with the name = '/members/' + `~self._name`
|
||||
and value = data in a given DCS.
|
||||
and value = connection_string in a given DCS.
|
||||
|
||||
:param data: json serialized information about instance (including connection strings)
|
||||
:param connection_string: how this instance can be accessed by other instances
|
||||
:param ttl: ttl for member key, optional parameter. If it is None `~self.member_ttl will be used`
|
||||
:returns: `!True` on success otherwise `!False`
|
||||
"""
|
||||
@@ -379,10 +263,6 @@ class AbstractDCS(object):
|
||||
def cancel_initialization(self):
|
||||
""" Removes the initialize key for a cluster """
|
||||
|
||||
@abc.abstractmethod
|
||||
def delete_cluster(self):
|
||||
"""Delete cluster from DCS"""
|
||||
|
||||
def watch(self, timeout):
|
||||
"""If the current node is a master it should just sleep.
|
||||
Any other node should watch for changes of leader key with a given timeout
|
||||
@@ -1,266 +0,0 @@
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import six
|
||||
|
||||
from consul import ConsulException, NotFound, base, std
|
||||
from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member
|
||||
from patroni.exceptions import DCSError
|
||||
from patroni.utils import sleep
|
||||
from requests.exceptions import RequestException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ConsulError(DCSError):
|
||||
pass
|
||||
|
||||
|
||||
class HTTPClient(std.HTTPClient):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(HTTPClient, self).__init__(*args, **kwargs)
|
||||
|
||||
def patch_default_timeout(self, timeout):
|
||||
# Set a default timeout for the `request.session.request` method, that is used
|
||||
# internally by the methods request.session.get, request.session.post and
|
||||
# others. We monkey-patch here to avoid reimplementing each individual method from
|
||||
# `std.HTTPClient`. By default, the timeout is not set. It means that a new
|
||||
# session may hang almost indefinitely waiting for the server to respond,
|
||||
# which is not what we want in Patroni.
|
||||
|
||||
request_func = getattr(self.session.request, '__func__' if six.PY3 else 'im_func')
|
||||
defaults_attr_name = '__defaults__' if six.PY3 else 'func_defaults'
|
||||
defaults = list(getattr(request_func, defaults_attr_name))
|
||||
code = request_func.__code__ if six.PY3 else request_func.func_code
|
||||
defaults[code.co_varnames[code.co_argcount - len(defaults):code.co_argcount].index('timeout')] = timeout
|
||||
setattr(request_func, defaults_attr_name, tuple(defaults)) # monkeypatching
|
||||
|
||||
def get(self, callback, path, params=None):
|
||||
# The get function is overridden to handle a special case of it being called
|
||||
# with an index and wait parameters. That form indicates that a user needs to
|
||||
# wait for the given key to change its value, with a wait timeout supplied. We
|
||||
# don't want our monkey-patched timeout to be less than the value of the wait
|
||||
# parameter, therefore, we set it to either the value of wait or a default of 5 minutes.
|
||||
|
||||
if isinstance(params, dict) and 'index' in params:
|
||||
timeout = (float(params['wait'][:-1]) if 'wait' in params else 300) + 1
|
||||
else:
|
||||
timeout = None
|
||||
return callback(self.response(self.session.get(self.uri(path, params), verify=self.verify, timeout=timeout)))
|
||||
|
||||
|
||||
class ConsulClient(base.Consul):
|
||||
|
||||
@staticmethod
|
||||
def connect(host, port, scheme, verify=True):
|
||||
return HTTPClient(host, port, scheme, verify)
|
||||
|
||||
|
||||
def catch_consul_errors(func):
|
||||
def wrapper(*args, **kwargs):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except (ConsulException, RequestException):
|
||||
return False
|
||||
return wrapper
|
||||
|
||||
|
||||
class Consul(AbstractDCS):
|
||||
|
||||
def __init__(self, config):
|
||||
super(Consul, self).__init__(config)
|
||||
self._ttl = None
|
||||
self._session = None
|
||||
self._my_member_data = None
|
||||
self.set_ttl(config.get('ttl') or 30)
|
||||
host, port = config.get('host', '127.0.0.1:8500').split(':')
|
||||
self._client = ConsulClient(host=host, port=port)
|
||||
self._client.http.patch_default_timeout(config['retry_timeout']/2.0)
|
||||
self._scope = config['scope']
|
||||
self.create_session()
|
||||
self.__do_not_watch = False
|
||||
|
||||
def create_session(self):
|
||||
while not self._session:
|
||||
try:
|
||||
self.refresh_session()
|
||||
except ConsulError:
|
||||
logger.info('waiting on consul')
|
||||
sleep(5)
|
||||
|
||||
def set_ttl(self, ttl):
|
||||
ttl = ttl/2.0 # My experiments have shown that session expires after 2*ttl time
|
||||
if self._ttl != ttl:
|
||||
self._session = None
|
||||
self.__do_not_watch = True
|
||||
self._ttl = ttl
|
||||
|
||||
def set_retry_timeout(self, retry_timeout):
|
||||
self._client.http.patch_default_timeout(retry_timeout/2.0)
|
||||
|
||||
def refresh_session(self):
|
||||
""":returns: `!True` if it had to create new session"""
|
||||
if self._session:
|
||||
try:
|
||||
return self._client.session.renew(self._session) is None
|
||||
except NotFound:
|
||||
self._session = None
|
||||
if not self._session:
|
||||
name = self._scope + '-' + self._name
|
||||
try:
|
||||
self._session = self._client.session.create(name=name, lock_delay=0, behavior='delete', ttl=self._ttl)
|
||||
except (ConsulException, RequestException):
|
||||
logger.exception('session.create')
|
||||
if not self._session:
|
||||
raise ConsulError('Failed to renew/create session')
|
||||
return True
|
||||
|
||||
def client_path(self, path):
|
||||
return super(Consul, self).client_path(path)[1:]
|
||||
|
||||
@staticmethod
|
||||
def member(node):
|
||||
return Member.from_node(node['ModifyIndex'], os.path.basename(node['Key']), node.get('Session'), node['Value'])
|
||||
|
||||
def _load_cluster(self):
|
||||
try:
|
||||
path = self.client_path('/')
|
||||
_, results = self._client.kv.get(path, recurse=True)
|
||||
|
||||
if results is None:
|
||||
raise NotFound
|
||||
|
||||
nodes = {}
|
||||
for node in results:
|
||||
node['Value'] = (node['Value'] or b'').decode('utf-8')
|
||||
nodes[os.path.relpath(node['Key'], path)] = node
|
||||
|
||||
# get initialize flag
|
||||
initialize = nodes.get(self._INITIALIZE)
|
||||
initialize = initialize and initialize['Value']
|
||||
|
||||
# get global dynamic configuration
|
||||
config = nodes.get(self._CONFIG)
|
||||
config = config and ClusterConfig.from_node(config['ModifyIndex'], config['Value'])
|
||||
|
||||
# get last leader operation
|
||||
last_leader_operation = nodes.get(self._LEADER_OPTIME)
|
||||
last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation['Value'])
|
||||
|
||||
# get list of members
|
||||
members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
|
||||
|
||||
# get leader
|
||||
leader = nodes.get(self._LEADER)
|
||||
if leader and leader['Value'] == self._name and self._session != leader.get('Session', 'x'):
|
||||
logger.info('I am leader but not owner of the session. Removing leader node')
|
||||
self._client.kv.delete(self.leader_path, cas=leader['ModifyIndex'])
|
||||
leader = None
|
||||
|
||||
if leader:
|
||||
member = Member(-1, leader['Value'], None, {})
|
||||
member = ([m for m in members if m.name == leader['Value']] or [member])[0]
|
||||
leader = Leader(leader['ModifyIndex'], leader.get('Session'), member)
|
||||
|
||||
# failover key
|
||||
failover = nodes.get(self._FAILOVER)
|
||||
if failover:
|
||||
failover = Failover.from_node(failover['ModifyIndex'], failover['Value'])
|
||||
|
||||
self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover)
|
||||
except NotFound:
|
||||
self._cluster = Cluster(False, None, None, None, [], None)
|
||||
except:
|
||||
logger.exception('get_cluster')
|
||||
raise ConsulError('Consul is not responding properly')
|
||||
|
||||
def touch_member(self, data, **kwargs):
|
||||
cluster = self.cluster
|
||||
member = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
|
||||
create_member = self.refresh_session()
|
||||
if member and (create_member or member.session != self._session):
|
||||
try:
|
||||
self._client.kv.delete(self.member_path)
|
||||
create_member = True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
if not create_member and member and data == self._my_member_data:
|
||||
return True
|
||||
|
||||
try:
|
||||
self._client.kv.put(self.member_path, data, acquire=self._session)
|
||||
self._my_member_data = data
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception('touch_member')
|
||||
return False
|
||||
|
||||
@catch_consul_errors
|
||||
def attempt_to_acquire_leader(self):
|
||||
ret = self._client.kv.put(self.leader_path, self._name, acquire=self._session)
|
||||
if not ret:
|
||||
logger.info('Could not take out TTL lock')
|
||||
return ret
|
||||
|
||||
def take_leader(self):
|
||||
return self.attempt_to_acquire_leader()
|
||||
|
||||
@catch_consul_errors
|
||||
def set_failover_value(self, value, index=None):
|
||||
return self._client.kv.put(self.failover_path, value, cas=index)
|
||||
|
||||
@catch_consul_errors
|
||||
def set_config_value(self, value, index=None):
|
||||
return self._client.kv.put(self.config_path, value, cas=index)
|
||||
|
||||
@catch_consul_errors
|
||||
def write_leader_optime(self, last_operation):
|
||||
return self._client.kv.put(self.leader_optime_path, last_operation)
|
||||
|
||||
@staticmethod
|
||||
def update_leader():
|
||||
return True
|
||||
|
||||
@catch_consul_errors
|
||||
def initialize(self, create_new=True, sysid=''):
|
||||
kwargs = {'cas': 0} if create_new else {}
|
||||
return self._client.kv.put(self.initialize_path, sysid, **kwargs)
|
||||
|
||||
@catch_consul_errors
|
||||
def cancel_initialization(self):
|
||||
return self._client.kv.delete(self.initialize_path)
|
||||
|
||||
@catch_consul_errors
|
||||
def delete_cluster(self):
|
||||
return self._client.kv.delete(self.client_path(''), recurse=True)
|
||||
|
||||
@catch_consul_errors
|
||||
def delete_leader(self):
|
||||
cluster = self.cluster
|
||||
if cluster and isinstance(cluster.leader, Leader) and cluster.leader.name == self._name:
|
||||
return self._client.kv.delete(self.leader_path, cas=cluster.leader.index)
|
||||
|
||||
def watch(self, timeout):
|
||||
if self.__do_not_watch:
|
||||
self.__do_not_watch = False
|
||||
return True
|
||||
|
||||
cluster = self.cluster
|
||||
if cluster and cluster.leader and cluster.leader.name != self._name and cluster.leader.index:
|
||||
end_time = time.time() + timeout
|
||||
while timeout >= 1:
|
||||
try:
|
||||
idx, _ = self._client.kv.get(self.leader_path, index=cluster.leader.index, wait=str(timeout) + 's')
|
||||
return str(idx) != str(cluster.leader.index)
|
||||
except (ConsulException, RequestException):
|
||||
logging.exception('watch')
|
||||
|
||||
timeout = end_time - time.time()
|
||||
|
||||
try:
|
||||
return super(Consul, self).watch(timeout)
|
||||
finally:
|
||||
self.event.clear()
|
||||
@@ -1,75 +0,0 @@
|
||||
import logging
|
||||
import random
|
||||
import requests
|
||||
import time
|
||||
|
||||
from patroni.dcs.zookeeper import ZooKeeper
|
||||
from patroni.utils import sleep
|
||||
from requests.exceptions import RequestException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ExhibitorEnsembleProvider(object):
|
||||
|
||||
TIMEOUT = 3.1
|
||||
|
||||
def __init__(self, hosts, port, uri_path='/exhibitor/v1/cluster/list', poll_interval=300):
|
||||
self._exhibitor_port = port
|
||||
self._uri_path = uri_path
|
||||
self._poll_interval = poll_interval
|
||||
self._exhibitors = hosts
|
||||
self._master_exhibitors = hosts
|
||||
self._zookeeper_hosts = ''
|
||||
self._next_poll = None
|
||||
while not self.poll():
|
||||
logger.info('waiting on exhibitor')
|
||||
sleep(5)
|
||||
|
||||
def poll(self):
|
||||
if self._next_poll and self._next_poll > time.time():
|
||||
return False
|
||||
|
||||
json = self._query_exhibitors(self._exhibitors)
|
||||
if not json:
|
||||
json = self._query_exhibitors(self._master_exhibitors)
|
||||
|
||||
if isinstance(json, dict) and 'servers' in json and 'port' in json:
|
||||
self._next_poll = time.time() + self._poll_interval
|
||||
zookeeper_hosts = ','.join([h + ':' + str(json['port']) for h in sorted(json['servers'])])
|
||||
if self._zookeeper_hosts != zookeeper_hosts:
|
||||
logger.info('ZooKeeper connection string has changed: %s => %s', self._zookeeper_hosts, zookeeper_hosts)
|
||||
self._zookeeper_hosts = zookeeper_hosts
|
||||
self._exhibitors = json['servers']
|
||||
return True
|
||||
return False
|
||||
|
||||
def _query_exhibitors(self, exhibitors):
|
||||
random.shuffle(exhibitors)
|
||||
for host in exhibitors:
|
||||
uri = 'http://{0}:{1}{2}'.format(host, self._exhibitor_port, self._uri_path)
|
||||
try:
|
||||
response = requests.get(uri, timeout=self.TIMEOUT)
|
||||
return response.json()
|
||||
except RequestException:
|
||||
pass
|
||||
return None
|
||||
|
||||
@property
|
||||
def zookeeper_hosts(self):
|
||||
return self._zookeeper_hosts
|
||||
|
||||
|
||||
class Exhibitor(ZooKeeper):
|
||||
|
||||
def __init__(self, config):
|
||||
interval = config.get('poll_interval', 300)
|
||||
self._ensemble_provider = ExhibitorEnsembleProvider(config['hosts'], config['port'], poll_interval=interval)
|
||||
config = config.copy()
|
||||
config['hosts'] = self._ensemble_provider.zookeeper_hosts
|
||||
super(Exhibitor, self).__init__(config)
|
||||
|
||||
def _load_cluster(self):
|
||||
if self._ensemble_provider.poll():
|
||||
self._client.set_hosts(self._ensemble_provider.zookeeper_hosts)
|
||||
return super(Exhibitor, self)._load_cluster()
|
||||
@@ -1,276 +0,0 @@
|
||||
import logging
|
||||
|
||||
from kazoo.client import KazooClient, KazooState
|
||||
from kazoo.exceptions import NoNodeError, NodeExistsError
|
||||
from kazoo.handlers.threading import SequentialThreadingHandler
|
||||
from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member
|
||||
from patroni.exceptions import DCSError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ZooKeeperError(DCSError):
|
||||
pass
|
||||
|
||||
|
||||
class PatroniSequentialThreadingHandler(SequentialThreadingHandler):
|
||||
|
||||
def __init__(self, connect_timeout):
|
||||
super(PatroniSequentialThreadingHandler, self).__init__()
|
||||
self.set_connect_timeout(connect_timeout)
|
||||
|
||||
def set_connect_timeout(self, connect_timeout):
|
||||
self._connect_timeout = max(1.0, connect_timeout/4.0)
|
||||
|
||||
def create_connection(self, *args, **kwargs):
|
||||
"""This method is trying to establish connection with one of the zookeeper nodes.
|
||||
Somehow strategy "fail earlier and retry more often" works way better comparing to
|
||||
the original strategy "try to connect with specified timeout".
|
||||
Since we want to try connect to zookeeper more often (with the smaller connect_timeout),
|
||||
he have to override `create_connection` method in the `SequentialThreadingHandler`
|
||||
class (which is used by `kazoo.Client`).
|
||||
|
||||
:param args: always contains `tuple(host, port)` as the first element and could contain
|
||||
`connect_timeout` (negotiated session timeout) as the second element."""
|
||||
|
||||
args = list(args)
|
||||
if len(args) == 1:
|
||||
args.append(self._connect_timeout)
|
||||
else:
|
||||
args[1] = max(self._connect_timeout, args[1]/10.0)
|
||||
return super(PatroniSequentialThreadingHandler, self).create_connection(*args, **kwargs)
|
||||
|
||||
|
||||
class ZooKeeper(AbstractDCS):
|
||||
|
||||
def __init__(self, config):
|
||||
super(ZooKeeper, self).__init__(config)
|
||||
|
||||
hosts = config.get('hosts', [])
|
||||
if isinstance(hosts, list):
|
||||
hosts = ','.join(hosts)
|
||||
|
||||
self._client = KazooClient(hosts, handler=PatroniSequentialThreadingHandler(config['retry_timeout']),
|
||||
timeout=config['ttl'], connection_retry={'max_delay': 1, 'max_tries': -1},
|
||||
command_retry={'deadline': config['retry_timeout'], 'max_delay': 1, 'max_tries': -1})
|
||||
self._client.add_listener(self.session_listener)
|
||||
|
||||
self._my_member_data = None
|
||||
self._fetch_cluster = True
|
||||
self._last_leader_operation = 0
|
||||
|
||||
self._client.start()
|
||||
|
||||
def session_listener(self, state):
|
||||
if state in [KazooState.SUSPENDED, KazooState.LOST]:
|
||||
self.cluster_watcher(None)
|
||||
|
||||
def cluster_watcher(self, event):
|
||||
self._fetch_cluster = True
|
||||
self.event.set()
|
||||
|
||||
def set_ttl(self, ttl):
|
||||
ttl = int(ttl * 1000)
|
||||
# I know, it's weird to access private attributes
|
||||
if self._client._session_timeout != ttl:
|
||||
self._client._session_timeout = ttl
|
||||
self._client.restart()
|
||||
|
||||
def set_retry_timeout(self, retry_timeout):
|
||||
self._client.handler.set_connect_timeout(retry_timeout)
|
||||
self._client._retry.deadline = retry_timeout
|
||||
|
||||
def get_node(self, key, watch=None):
|
||||
try:
|
||||
ret = self._client.get(key, watch)
|
||||
return (ret[0].decode('utf-8'), ret[1])
|
||||
except NoNodeError:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def member(name, value, znode):
|
||||
return Member.from_node(znode.version, name, znode.ephemeralOwner, value)
|
||||
|
||||
def get_children(self, key, watch=None):
|
||||
try:
|
||||
return self._client.get_children(key, watch)
|
||||
except NoNodeError:
|
||||
return []
|
||||
|
||||
def load_members(self):
|
||||
members = []
|
||||
for member in self.get_children(self.members_path, self.cluster_watcher):
|
||||
data = self.get_node(self.members_path + member)
|
||||
if data is not None:
|
||||
members.append(self.member(member, *data))
|
||||
return members
|
||||
|
||||
def _inner_load_cluster(self):
|
||||
self._fetch_cluster = False
|
||||
self.event.clear()
|
||||
nodes = set(self.get_children(self.client_path(''), self.cluster_watcher))
|
||||
if not nodes:
|
||||
self._fetch_cluster = True
|
||||
|
||||
# get initialize flag
|
||||
initialize = (self.get_node(self.initialize_path) or [None])[0] if self._INITIALIZE in nodes else None
|
||||
|
||||
# get global dynamic configuration
|
||||
config = self.get_node(self.config_path, watch=self.cluster_watcher) if self._CONFIG in nodes else None
|
||||
config = config and ClusterConfig.from_node(config[1].version, config[0], config[1].mzxid)
|
||||
|
||||
# get list of members
|
||||
members = self.load_members() if self._MEMBERS[:-1] in nodes else []
|
||||
|
||||
# get leader
|
||||
leader = self.get_node(self.leader_path) if self._LEADER in nodes else None
|
||||
if leader:
|
||||
client_id = self._client.client_id
|
||||
if leader[0] == self._name and client_id is not None and client_id[0] != leader[1].ephemeralOwner:
|
||||
logger.info('I am leader but not owner of the session. Removing leader node')
|
||||
self._client.delete(self.leader_path)
|
||||
leader = None
|
||||
|
||||
if leader:
|
||||
member = Member(-1, leader[0], None, {})
|
||||
member = ([m for m in members if m.name == leader[0]] or [member])[0]
|
||||
leader = Leader(leader[1].version, leader[1].ephemeralOwner, member)
|
||||
self._fetch_cluster = member.index == -1
|
||||
|
||||
# failover key
|
||||
failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
|
||||
failover = failover and Failover.from_node(failover[1].version, failover[0])
|
||||
|
||||
# get last leader operation
|
||||
optime = self.get_node(self.leader_optime_path) if self._OPTIME in nodes and self._fetch_cluster else None
|
||||
self._last_leader_operation = 0 if optime is None else int(optime[0])
|
||||
self._cluster = Cluster(initialize, config, leader, self._last_leader_operation, members, failover)
|
||||
|
||||
def _load_cluster(self):
|
||||
if self._fetch_cluster or self._cluster is None:
|
||||
try:
|
||||
self._client.retry(self._inner_load_cluster)
|
||||
except:
|
||||
logger.exception('get_cluster')
|
||||
self.cluster_watcher(None)
|
||||
raise ZooKeeperError('ZooKeeper in not responding properly')
|
||||
|
||||
def _create(self, path, value, **kwargs):
|
||||
try:
|
||||
self._client.retry(self._client.create, path, value.encode('utf-8'), **kwargs)
|
||||
return True
|
||||
except:
|
||||
return False
|
||||
|
||||
def attempt_to_acquire_leader(self):
|
||||
ret = self._create(self.leader_path, self._name, makepath=True, ephemeral=True)
|
||||
if not ret:
|
||||
logger.info('Could not take out TTL lock')
|
||||
return ret
|
||||
|
||||
def set_failover_value(self, value, index=None):
|
||||
try:
|
||||
self._client.retry(self._client.set, self.failover_path, value.encode('utf-8'), version=index or -1)
|
||||
return True
|
||||
except NoNodeError:
|
||||
return value == '' or (index is None and self._create(self.failover_path, value))
|
||||
except:
|
||||
logging.exception('set_failover_value')
|
||||
return False
|
||||
|
||||
def set_config_value(self, value, index=None):
|
||||
try:
|
||||
self._client.retry(self._client.set, self.config_path, value.encode('utf-8'), version=index or -1)
|
||||
return True
|
||||
except NoNodeError:
|
||||
return index is None and self._create(self.config_path, value)
|
||||
except Exception:
|
||||
logging.exception('set_config_value')
|
||||
return False
|
||||
|
||||
def initialize(self, create_new=True, sysid=""):
|
||||
return self._create(self.initialize_path, sysid, makepath=True) if create_new \
|
||||
else self._client.retry(self._client.set, self.initialize_path, sysid.encode("utf-8"))
|
||||
|
||||
def touch_member(self, data, ttl=None):
|
||||
cluster = self.cluster
|
||||
member = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
|
||||
path = self.member_path
|
||||
data = data.encode('utf-8')
|
||||
if member and self._client.client_id is not None and member.session != self._client.client_id[0]:
|
||||
try:
|
||||
self._client.retry(self._client.delete, path)
|
||||
except NoNodeError:
|
||||
pass
|
||||
except:
|
||||
return False
|
||||
member = None
|
||||
|
||||
if member and data == self._my_member_data:
|
||||
return True
|
||||
|
||||
try:
|
||||
if member:
|
||||
self._client.retry(self._client.set, path, data)
|
||||
else:
|
||||
self._client.retry(self._client.create, path, data, makepath=True, ephemeral=True)
|
||||
self._my_member_data = data
|
||||
return True
|
||||
except NodeExistsError:
|
||||
try:
|
||||
self._client.retry(self._client.set, path, data)
|
||||
self._my_member_data = data
|
||||
return True
|
||||
except:
|
||||
logger.exception('touch_member')
|
||||
except:
|
||||
logger.exception('touch_member')
|
||||
return False
|
||||
|
||||
def take_leader(self):
|
||||
return self.attempt_to_acquire_leader()
|
||||
|
||||
def write_leader_optime(self, last_operation):
|
||||
last_operation = last_operation.encode('utf-8')
|
||||
if last_operation != self._last_leader_operation:
|
||||
self._last_leader_operation = last_operation
|
||||
path = self.leader_optime_path
|
||||
try:
|
||||
self._client.retry(self._client.set, path, last_operation)
|
||||
except NoNodeError:
|
||||
try:
|
||||
self._client.retry(self._client.create, path, last_operation, makepath=True)
|
||||
except:
|
||||
logger.exception('Failed to create %s', path)
|
||||
except:
|
||||
logger.exception('Failed to update %s', path)
|
||||
|
||||
def update_leader(self):
|
||||
return True
|
||||
|
||||
def delete_leader(self):
|
||||
self._client.restart()
|
||||
self._my_member_data = None
|
||||
return True
|
||||
|
||||
def _cancel_initialization(self):
|
||||
node = self.get_node(self.initialize_path)
|
||||
if node:
|
||||
self._client.delete(self.initialize_path, version=node[1].version)
|
||||
|
||||
def cancel_initialization(self):
|
||||
try:
|
||||
self._client.retry(self._cancel_initialization)
|
||||
except:
|
||||
logger.exception("Unable to delete initialize key")
|
||||
|
||||
def delete_cluster(self):
|
||||
try:
|
||||
return self._client.retry(self._client.delete, self.client_path(''), recursive=True)
|
||||
except NoNodeError:
|
||||
return True
|
||||
|
||||
def watch(self, timeout):
|
||||
if super(ZooKeeper, self).watch(timeout):
|
||||
self._fetch_cluster = True
|
||||
return self._fetch_cluster
|
||||
@@ -6,15 +6,14 @@ import random
|
||||
import requests
|
||||
import socket
|
||||
import time
|
||||
import urllib3
|
||||
|
||||
from dns.exception import DNSException
|
||||
from dns import resolver
|
||||
from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member
|
||||
from patroni.dcs import AbstractDCS, Cluster, Failover, Leader, Member
|
||||
from patroni.exceptions import DCSError
|
||||
from patroni.utils import Retry, RetryFailedError, sleep
|
||||
from urllib3.exceptions import HTTPError, ReadTimeoutError
|
||||
from requests.exceptions import RequestException
|
||||
from six.moves.http_client import HTTPException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -26,7 +25,7 @@ class EtcdError(DCSError):
|
||||
class Client(etcd.Client):
|
||||
|
||||
def __init__(self, config):
|
||||
super(Client, self).__init__(read_timeout=config['retry_timeout'])
|
||||
super(Client, self).__init__(read_timeout=5)
|
||||
self._config = config
|
||||
self._load_machines_cache()
|
||||
self._allow_reconnect = True
|
||||
@@ -50,70 +49,11 @@ class Client(etcd.Client):
|
||||
self._update_machines_cache = True
|
||||
return [self._base_uri]
|
||||
|
||||
def set_read_timeout(self, timeout):
|
||||
self._read_timeout = timeout
|
||||
|
||||
def _do_http_request(self, request_executor, method, url, fields=None, **kwargs):
|
||||
try:
|
||||
response = request_executor(method, url, fields=fields, **kwargs)
|
||||
response.data.decode('utf-8')
|
||||
self._check_cluster_id(response)
|
||||
except (HTTPError, HTTPException, socket.error, socket.timeout) as e:
|
||||
if (isinstance(fields, dict) and fields.get("wait") == "true" and
|
||||
isinstance(e, ReadTimeoutError)):
|
||||
logger.debug("Watch timed out.")
|
||||
raise etcd.EtcdWatchTimedOut("Watch timed out: {0}".format(e), cause=e)
|
||||
logger.error("Request to server %s failed: %r", self._base_uri, e)
|
||||
logger.info("Reconnection allowed, looking for another server.")
|
||||
self._base_uri = self._next_server(cause=e)
|
||||
response = False
|
||||
return response
|
||||
|
||||
def api_execute(self, path, method, params=None, timeout=None):
|
||||
if not path.startswith('/'):
|
||||
raise ValueError('Path does not start with /')
|
||||
|
||||
kwargs = {'fields': params, 'redirect': self.allow_redirect,
|
||||
'headers': self._get_headers(), 'preload_content': False}
|
||||
|
||||
if method in [self._MGET, self._MDELETE]:
|
||||
request_executor = self.http.request
|
||||
elif method in [self._MPUT, self._MPOST]:
|
||||
request_executor = self.http.request_encode_body
|
||||
kwargs['encode_multipart'] = False
|
||||
else:
|
||||
raise etcd.EtcdException('HTTP method {0} not supported'.format(method))
|
||||
|
||||
def api_execute(self, path, method, **kwargs):
|
||||
# Update machines_cache if previous attempt of update has failed
|
||||
if self._update_machines_cache:
|
||||
self._load_machines_cache()
|
||||
|
||||
if timeout is None:
|
||||
# calculate the number of retries and timeout *per node*
|
||||
# actual number of retries depends on the number of nodes
|
||||
etcd_nodes = len(self._machines_cache) + 1
|
||||
kwargs['retries'] = 0 if etcd_nodes > 3 else (1 if etcd_nodes > 1 else 2)
|
||||
|
||||
# if etcd_nodes > 3:
|
||||
# kwargs.update({'retries': 0, 'timeout': float(self.read_timeout)/etcd_nodes})
|
||||
# elif etcd_nodes > 1:
|
||||
# kwargs.update({'retries': 1, 'timeout': self.read_timeout/2.0/etcd_nodes})
|
||||
# else:
|
||||
# kwargs.update({'retries': 2, 'timeout': self.read_timeout/3.0})
|
||||
kwargs['timeout'] = self.read_timeout/float(kwargs['retries'] + 1)/etcd_nodes
|
||||
else:
|
||||
kwargs.update({'retries': 0, 'timeout': timeout})
|
||||
|
||||
response = False
|
||||
|
||||
self._update_machines_cache and self._load_machines_cache()
|
||||
try:
|
||||
while not response:
|
||||
response = self._do_http_request(request_executor, method, self._base_uri + path, **kwargs)
|
||||
|
||||
if response is False and not self._use_proxies:
|
||||
self._machines_cache = self.machines
|
||||
self._machines_cache.remove(self._base_uri)
|
||||
return self._handle_server_response(response)
|
||||
return super(Client, self).api_execute(path, method, **kwargs)
|
||||
except etcd.EtcdConnectionFailed:
|
||||
self._update_machines_cache = True
|
||||
raise
|
||||
@@ -126,6 +66,16 @@ class Client(etcd.Client):
|
||||
logger.exception('Can not resolve SRV for %s', host)
|
||||
return []
|
||||
|
||||
# try to workarond bug in python-etcd: https://github.com/jplana/python-etcd/issues/81
|
||||
def _result_from_response(self, response):
|
||||
try:
|
||||
response.data.decode('utf-8')
|
||||
except urllib3.exceptions.TimeoutError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise etcd.EtcdException('Unable to decode server response: %s' % e)
|
||||
return super(Client, self)._result_from_response(response)
|
||||
|
||||
def _get_machines_cache_from_srv(self, discovery_srv):
|
||||
"""Fetch list of etcd-cluster member by resolving _etcd-server._tcp. SRV record.
|
||||
This record should contain list of host and peer ports which could be used to run
|
||||
@@ -133,9 +83,9 @@ class Client(etcd.Client):
|
||||
|
||||
ret = []
|
||||
for host, port in self.get_srv_record(discovery_srv):
|
||||
url = '{0}://{1}:{2}/members'.format(self._protocol, host, port)
|
||||
url = '{}://{}:{}/members'.format(self._protocol, host, port)
|
||||
try:
|
||||
response = requests.get(url, timeout=self.read_timeout)
|
||||
response = requests.get(url, timeout=5)
|
||||
if response.ok:
|
||||
for member in response.json():
|
||||
ret.extend(member['clientURLs'])
|
||||
@@ -151,10 +101,10 @@ class Client(etcd.Client):
|
||||
host, port = addr.split(':')
|
||||
try:
|
||||
for r in set(socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)):
|
||||
ret.append('{0}://{1}:{2}'.format(self._protocol, r[4][0], r[4][1]))
|
||||
ret.append('{}://{}:{}'.format(self._protocol, r[4][0], r[4][1]))
|
||||
except socket.error:
|
||||
logger.exception('Can not resolve %s', host)
|
||||
return list(set(ret)) if ret else ['{0}://{1}:{2}'.format(self._protocol, host, port)]
|
||||
return list(set(ret)) if ret else ['{}://{}:{}'.format(self._protocol, host, port)]
|
||||
|
||||
def _load_machines_cache(self):
|
||||
"""This method should fill up `_machines_cache` from scratch.
|
||||
@@ -182,9 +132,7 @@ class Client(etcd.Client):
|
||||
# After filling up initial list of machines_cache we should ask etcd-cluster about actual list
|
||||
self._base_uri = self._machines_cache.pop(0)
|
||||
self._machines_cache = self.machines
|
||||
|
||||
if self._base_uri in self._machines_cache:
|
||||
self._machines_cache.remove(self._base_uri)
|
||||
self._base_uri in self._machines_cache and self._machines_cache.remove(self._base_uri)
|
||||
|
||||
self._update_machines_cache = False
|
||||
|
||||
@@ -192,7 +140,7 @@ class Client(etcd.Client):
|
||||
def catch_etcd_errors(func):
|
||||
def wrapper(*args, **kwargs):
|
||||
try:
|
||||
return func(*args, **kwargs) is not None
|
||||
return not func(*args, **kwargs) is None
|
||||
except (RetryFailedError, etcd.EtcdException):
|
||||
return False
|
||||
except:
|
||||
@@ -204,21 +152,20 @@ def catch_etcd_errors(func):
|
||||
|
||||
class Etcd(AbstractDCS):
|
||||
|
||||
def __init__(self, config):
|
||||
super(Etcd, self).__init__(config)
|
||||
self._ttl = int(config.get('ttl') or 30)
|
||||
self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
|
||||
retry_exceptions=(etcd.EtcdLeaderElectionInProgress,
|
||||
def __init__(self, name, config):
|
||||
super(Etcd, self).__init__(name, config)
|
||||
self.ttl = config.get('ttl', 30)
|
||||
self._retry = Retry(deadline=10, max_delay=1, max_tries=-1,
|
||||
retry_exceptions=(etcd.EtcdConnectionFailed,
|
||||
etcd.EtcdLeaderElectionInProgress,
|
||||
etcd.EtcdWatcherCleared,
|
||||
etcd.EtcdEventIndexCleared))
|
||||
self._client = self.get_etcd_client(config)
|
||||
self.__do_not_watch = False
|
||||
self.client = self.get_etcd_client(config)
|
||||
|
||||
def retry(self, *args, **kwargs):
|
||||
return self._retry.copy()(*args, **kwargs)
|
||||
|
||||
@staticmethod
|
||||
def get_etcd_client(config):
|
||||
def get_etcd_client(self, config):
|
||||
client = None
|
||||
while not client:
|
||||
try:
|
||||
@@ -228,69 +175,56 @@ class Etcd(AbstractDCS):
|
||||
sleep(5)
|
||||
return client
|
||||
|
||||
def set_ttl(self, ttl):
|
||||
ttl = int(ttl)
|
||||
self.__do_not_watch = self._ttl != ttl
|
||||
self._ttl = ttl
|
||||
|
||||
def set_retry_timeout(self, retry_timeout):
|
||||
self._retry.deadline = retry_timeout
|
||||
self._client.set_read_timeout(retry_timeout)
|
||||
|
||||
@staticmethod
|
||||
def member(node):
|
||||
return Member.from_node(node.modifiedIndex, os.path.basename(node.key), node.ttl, node.value)
|
||||
|
||||
def _load_cluster(self):
|
||||
try:
|
||||
result = self.retry(self._client.read, self.client_path(''), recursive=True)
|
||||
result = self.retry(self.client.read, self.client_path(''), recursive=True)
|
||||
nodes = {os.path.relpath(node.key, result.key): node for node in result.leaves}
|
||||
|
||||
# get initialize flag
|
||||
initialize = nodes.get(self._INITIALIZE)
|
||||
initialize = nodes.get(self._INITIALIZE, None)
|
||||
initialize = initialize and initialize.value
|
||||
|
||||
# get global dynamic configuration
|
||||
config = nodes.get(self._CONFIG)
|
||||
config = config and ClusterConfig.from_node(config.modifiedIndex, config.value)
|
||||
|
||||
# get last leader operation
|
||||
last_leader_operation = nodes.get(self._LEADER_OPTIME)
|
||||
last_leader_operation = nodes.get(self._LEADER_OPTIME, None)
|
||||
last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation.value)
|
||||
|
||||
# get list of members
|
||||
members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
|
||||
|
||||
# get leader
|
||||
leader = nodes.get(self._LEADER)
|
||||
leader = nodes.get(self._LEADER, None)
|
||||
if leader:
|
||||
member = Member(-1, leader.value, None, {})
|
||||
member = ([m for m in members if m.name == leader.value] or [member])[0]
|
||||
leader = Leader(leader.modifiedIndex, leader.ttl, member)
|
||||
|
||||
# failover key
|
||||
failover = nodes.get(self._FAILOVER)
|
||||
failover = nodes.get(self._FAILOVER, None)
|
||||
if failover:
|
||||
failover = Failover.from_node(failover.modifiedIndex, failover.value)
|
||||
|
||||
self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover)
|
||||
self._cluster = Cluster(initialize, leader, last_leader_operation, members, failover)
|
||||
except etcd.EtcdKeyNotFound:
|
||||
self._cluster = Cluster(False, None, None, None, [], None)
|
||||
self._cluster = Cluster(False, None, None, [], None)
|
||||
except:
|
||||
logger.exception('get_cluster')
|
||||
raise EtcdError('Etcd is not responding properly')
|
||||
|
||||
@catch_etcd_errors
|
||||
def touch_member(self, data, ttl=None):
|
||||
return self.retry(self._client.set, self.member_path, data, ttl or self._ttl)
|
||||
def touch_member(self, connection_string, ttl=None):
|
||||
return self.retry(self.client.set, self.member_path, connection_string, ttl or self.ttl)
|
||||
|
||||
@catch_etcd_errors
|
||||
def take_leader(self):
|
||||
return self.retry(self._client.set, self.leader_path, self._name, self._ttl)
|
||||
return self.retry(self.client.set, self.leader_path, self._name, self.ttl)
|
||||
|
||||
def attempt_to_acquire_leader(self):
|
||||
try:
|
||||
return bool(self.retry(self._client.write, self.leader_path, self._name, ttl=self._ttl, prevExist=False))
|
||||
return bool(self.retry(self.client.write, self.leader_path, self._name, ttl=self.ttl, prevExist=False))
|
||||
except etcd.EtcdAlreadyExist:
|
||||
logger.info('Could not take out TTL lock')
|
||||
except (RetryFailedError, etcd.EtcdException):
|
||||
@@ -299,54 +233,43 @@ class Etcd(AbstractDCS):
|
||||
|
||||
@catch_etcd_errors
|
||||
def set_failover_value(self, value, index=None):
|
||||
return self._client.write(self.failover_path, value, prevIndex=index or 0)
|
||||
|
||||
@catch_etcd_errors
|
||||
def set_config_value(self, value, index=None):
|
||||
return self._client.write(self.config_path, value, prevIndex=index or 0)
|
||||
return self.client.write(self.failover_path, value, prevIndex=index or 0)
|
||||
|
||||
@catch_etcd_errors
|
||||
def write_leader_optime(self, last_operation):
|
||||
return self._client.set(self.leader_optime_path, last_operation)
|
||||
return self.client.set(self.leader_optime_path, last_operation)
|
||||
|
||||
@catch_etcd_errors
|
||||
def update_leader(self):
|
||||
return self.retry(self._client.test_and_set, self.leader_path, self._name, self._name, self._ttl)
|
||||
return self.retry(self.client.test_and_set, self.leader_path, self._name, self._name, self.ttl)
|
||||
|
||||
@catch_etcd_errors
|
||||
def initialize(self, create_new=True, sysid=""):
|
||||
return self.retry(self._client.write, self.initialize_path, sysid, prevExist=(not create_new))
|
||||
return self.retry(self.client.write, self.initialize_path, sysid, prevExist=(not create_new))
|
||||
|
||||
@catch_etcd_errors
|
||||
def delete_leader(self):
|
||||
return self._client.delete(self.leader_path, prevValue=self._name)
|
||||
return self.client.delete(self.leader_path, prevValue=self._name)
|
||||
|
||||
@catch_etcd_errors
|
||||
def cancel_initialization(self):
|
||||
return self.retry(self._client.delete, self.initialize_path)
|
||||
|
||||
@catch_etcd_errors
|
||||
def delete_cluster(self):
|
||||
return self.retry(self._client.delete, self.client_path(''), recursive=True)
|
||||
return self.retry(self.client.delete, self.initialize_path)
|
||||
|
||||
def watch(self, timeout):
|
||||
if self.__do_not_watch:
|
||||
self.__do_not_watch = False
|
||||
return True
|
||||
|
||||
cluster = self.cluster
|
||||
# watch on leader key changes if it is defined and current node is not lock owner
|
||||
if cluster and cluster.leader and cluster.leader.name != self._name and cluster.leader.index:
|
||||
if cluster and cluster.leader and cluster.leader.name != self._name:
|
||||
end_time = time.time() + timeout
|
||||
index = cluster.leader.index
|
||||
|
||||
while timeout >= 1: # when timeout is too small urllib3 doesn't have enough time to connect
|
||||
while index and timeout >= 1: # when timeout is too small urllib3 doesn't have enough time to connect
|
||||
try:
|
||||
self._client.watch(self.leader_path, index=cluster.leader.index + 1, timeout=timeout + 0.5)
|
||||
self.client.watch(self.leader_path, index=index + 1, timeout=timeout + 0.5)
|
||||
# Synchronous work of all cluster members with etcd is less expensive
|
||||
# than reestablishing http connection every time from every replica.
|
||||
return True
|
||||
except etcd.EtcdWatchTimedOut:
|
||||
self._client.http.clear()
|
||||
except urllib3.exceptions.TimeoutError:
|
||||
self.client.http.clear()
|
||||
return False
|
||||
except etcd.EtcdException:
|
||||
logging.exception('watch')
|
||||
@@ -13,6 +13,10 @@ class PatroniException(Exception):
|
||||
return repr(self.value)
|
||||
|
||||
|
||||
class PatroniCtlException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class PostgresException(PatroniException):
|
||||
pass
|
||||
|
||||
|
||||
+190
-163
@@ -3,18 +3,15 @@ import logging
|
||||
import psycopg2
|
||||
import requests
|
||||
import sys
|
||||
import datetime
|
||||
import pytz
|
||||
|
||||
from multiprocessing.pool import ThreadPool
|
||||
from patroni.async_executor import AsyncExecutor
|
||||
from patroni.exceptions import DCSError, PostgresConnectionException
|
||||
from patroni.utils import sleep
|
||||
from multiprocessing.pool import ThreadPool
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Ha(object):
|
||||
class Ha:
|
||||
|
||||
def __init__(self, patroni):
|
||||
self.patroni = patroni
|
||||
@@ -22,13 +19,17 @@ class Ha(object):
|
||||
self.dcs = patroni.dcs
|
||||
self.cluster = None
|
||||
self.old_cluster = None
|
||||
self.recovering = False
|
||||
self._async_executor = AsyncExecutor()
|
||||
self.bdr = patroni.bdr
|
||||
self.use_bdr = self.bdr and\
|
||||
self.bdr.get('enable') and\
|
||||
(self.bdr.get('database') is not None)
|
||||
self.bdr_may_need_reinitialize = False
|
||||
|
||||
def load_cluster_from_dcs(self):
|
||||
cluster = self.dcs.get_cluster()
|
||||
|
||||
# We want to keep the state of cluster when it was healthy
|
||||
# We want to keep the state of cluster when it was healhy
|
||||
if not cluster.is_unlocked() or not self.old_cluster:
|
||||
self.old_cluster = cluster
|
||||
self.cluster = cluster
|
||||
@@ -38,7 +39,7 @@ class Ha(object):
|
||||
|
||||
def update_lock(self):
|
||||
ret = self.dcs.update_leader()
|
||||
if ret and not self._async_executor.busy:
|
||||
if ret:
|
||||
try:
|
||||
self.dcs.write_leader_optime(self.state_handler.last_operation())
|
||||
except:
|
||||
@@ -50,47 +51,52 @@ class Ha(object):
|
||||
logger.info('Lock owner: %s; I am %s', lock_owner, self.state_handler.name)
|
||||
return lock_owner == self.state_handler.name
|
||||
|
||||
def touch_member(self):
|
||||
def touch_member(self, ttl=None):
|
||||
data = {
|
||||
'conn_url': self.state_handler.connection_string,
|
||||
'api_url': self.patroni.api.connection_string,
|
||||
'state': self.state_handler.state,
|
||||
'role': self.state_handler.role
|
||||
'role': self.state_handler.role,
|
||||
'tags': self.patroni.tags
|
||||
}
|
||||
if self.patroni.tags:
|
||||
data['tags'] = self.patroni.tags
|
||||
if self.state_handler.pending_restart:
|
||||
data['pending_restart'] = True
|
||||
if not self._async_executor.busy and data['state'] in ['running', 'restarting', 'starting']:
|
||||
if data['state'] in ['running', 'restarting', 'starting']:
|
||||
try:
|
||||
data['xlog_location'] = self.state_handler.xlog_position()
|
||||
except:
|
||||
pass
|
||||
self.dcs.touch_member(json.dumps(data, separators=(',', ':')))
|
||||
self.dcs.touch_member(json.dumps(data, separators=(',', ':')), ttl=ttl)
|
||||
|
||||
def clone(self, clone_member=None, msg='(without leader)'):
|
||||
if self.state_handler.clone(clone_member):
|
||||
logger.info('bootstrapped %s', msg)
|
||||
cluster = self.dcs.get_cluster()
|
||||
node_to_follow = self._get_node_to_follow(cluster)
|
||||
self.state_handler.follow(node_to_follow, cluster.leader, True)
|
||||
def copy_backup_from_leader(self, leader):
|
||||
if self.state_handler.bootstrap(leader):
|
||||
logger.info('bootstrapped from leader')
|
||||
else:
|
||||
logger.error('failed to bootstrap %s', msg)
|
||||
self.state_handler.stop('immediate')
|
||||
self.state_handler.remove_data_directory()
|
||||
logger.error('failed to bootstrap from leader')
|
||||
|
||||
def bootstrap(self):
|
||||
def initialize_bdr(self, leader):
|
||||
if self.state_handler.initialize_bdr_database(leader):
|
||||
logger.info("bootstrapped from leader")
|
||||
else:
|
||||
self.state_handler.stop('immediate')
|
||||
self.state_handler.remove_data_directory()
|
||||
logger.error('failed to bootstrap from leader')
|
||||
|
||||
def bootstrap(self, bdr=False):
|
||||
if not self.cluster.is_unlocked(): # cluster already has leader
|
||||
clone_member = self.cluster.get_clone_member()
|
||||
member_role = 'leader' if clone_member == self.cluster.leader else 'replica'
|
||||
msg = "from {0} '{1}'".format(member_role, clone_member.name)
|
||||
self._async_executor.schedule('bootstrap {0}'.format(msg))
|
||||
self._async_executor.run_async(self.clone, args=(clone_member, msg))
|
||||
return 'trying to bootstrap {0}'.format(msg)
|
||||
# no initialize key and node is allowed to be master and has 'bootstrap' section in a configuration file
|
||||
elif self.cluster.initialize is None and not self.patroni.nofailover and 'bootstrap' in self.patroni.config:
|
||||
self._async_executor.schedule('bootstrap from leader')
|
||||
if not bdr:
|
||||
self._async_executor.run_async(self.copy_backup_from_leader, args=(self.cluster.leader, ))
|
||||
else:
|
||||
self._async_executor.run_async(self.initialize_bdr, args=(self.cluster.leader, ))
|
||||
return 'trying to bootstrap from leader'
|
||||
elif not self.cluster.initialize and not self.patroni.nofailover: # no initialize key
|
||||
if self.dcs.initialize(create_new=True): # race for initialization
|
||||
try:
|
||||
self.state_handler.bootstrap(self.patroni.config['bootstrap'])
|
||||
if not bdr:
|
||||
self.state_handler.bootstrap()
|
||||
else:
|
||||
self.state_handler.initialize_bdr_database()
|
||||
self.dcs.initialize(create_new=False, sysid=self.state_handler.sysid)
|
||||
except: # initdb or start failed
|
||||
# remove initialization key and give a chance to other members
|
||||
@@ -99,46 +105,65 @@ class Ha(object):
|
||||
self.state_handler.stop('immediate')
|
||||
self.state_handler.move_data_directory()
|
||||
raise
|
||||
self.dcs.set_config_value(json.dumps(self.patroni.config.dynamic_configuration, separators=(',', ':')))
|
||||
self.dcs.take_leader()
|
||||
self.load_cluster_from_dcs()
|
||||
return 'initialized a new cluster'
|
||||
else:
|
||||
return 'failed to acquire initialize lock'
|
||||
else:
|
||||
if self.state_handler.can_create_replica_without_replication_connection():
|
||||
self._async_executor.run_async(self.clone)
|
||||
return "trying to bootstrap (without leader)"
|
||||
return 'waiting for leader to bootstrap'
|
||||
|
||||
def bootstrap_bdr(self):
|
||||
return self.bootstrap(bdr=True)
|
||||
|
||||
def recover(self):
|
||||
self.recovering = True
|
||||
return self.follow("starting as readonly because i had the session lock", "starting as a secondary", True, True)
|
||||
has_lock = self.has_lock()
|
||||
|
||||
def _get_node_to_follow(self, cluster):
|
||||
# determine the node to follow. If replicatefrom tag is set,
|
||||
# try to follow the node mentioned there, otherwise, follow the leader.
|
||||
if not self.patroni.replicatefrom or self.patroni.replicatefrom == self.state_handler.name:
|
||||
node_to_follow = cluster.leader
|
||||
else:
|
||||
node_to_follow = cluster.get_member(self.patroni.replicatefrom)
|
||||
# try to see if we are the former master that crashed. If so - we likely need to run pg_rewind
|
||||
# in order to join the former standby being promoted.
|
||||
pg_controldata = self.state_handler.controldata()
|
||||
if not has_lock and pg_controldata and\
|
||||
pg_controldata.get('Database cluster state', '') == 'in production': # crashed master
|
||||
self.state_handler.require_rewind()
|
||||
|
||||
return node_to_follow if node_to_follow and node_to_follow.name != self.state_handler.name else None
|
||||
# XXX: follow the leader calls stop, which might take quite some time.
|
||||
# perhaps we should run sync asynchronously
|
||||
# (we still need the exit code from follow_the_leader)
|
||||
ret = self.state_handler.follow_the_leader(None if has_lock else self.cluster.leader, recovery=True)
|
||||
if not ret:
|
||||
if not has_lock:
|
||||
return 'failed to start postgres'
|
||||
self.dcs.delete_leader()
|
||||
self.dcs.reset_cluster()
|
||||
return 'removed leader key after trying and failing to start postgres'
|
||||
if not has_lock:
|
||||
return 'started as a secondary'
|
||||
logger.info('started as readonly because i had the session lock')
|
||||
self.load_cluster_from_dcs()
|
||||
|
||||
def follow(self, demote_reason, follow_reason, refresh=True, recovery=False):
|
||||
if refresh:
|
||||
self.load_cluster_from_dcs()
|
||||
def recover_bdr(self):
|
||||
if not self.bdr_may_need_reinitialize:
|
||||
self.state_handler.start_bdr()
|
||||
return "started as a member of BDR group"
|
||||
has_lock = self.has_lock()
|
||||
if self.cluster.is_unlocked(): # no leader yet, and we need to reinitialize from the leader
|
||||
return "waiting for another member of the BDR group"
|
||||
if has_lock:
|
||||
self.dcs.delete_leader()
|
||||
return "released the lock to another member of the BDR group"
|
||||
if self.state_handler.initialize_bdr_database(self.cluster.leader, empty=False):
|
||||
# we reinitialized this node, make sure the cluster is aware of it.
|
||||
self.touch_member()
|
||||
self.bdr_may_need_reinitialize = False
|
||||
return "reinitialized and started as a member of the BDR group"
|
||||
|
||||
if recovery:
|
||||
ret = demote_reason if self.has_lock() else follow_reason
|
||||
else:
|
||||
ret = demote_reason if self.state_handler.is_leader() else follow_reason
|
||||
|
||||
node_to_follow = self._get_node_to_follow(self.cluster)
|
||||
|
||||
if not self.state_handler.check_recovery_conf(node_to_follow) or recovery:
|
||||
def follow_the_leader(self, demote_reason, follow_reason, refresh=True):
|
||||
refresh and self.load_cluster_from_dcs()
|
||||
ret = demote_reason if self.state_handler.is_leader() else follow_reason
|
||||
leader = self.cluster.leader
|
||||
leader = None if (leader and leader.name) == self.state_handler.name else leader
|
||||
if not self.state_handler.check_recovery_conf(leader):
|
||||
self._async_executor.schedule('changing primary_conninfo and restarting')
|
||||
self._async_executor.run_async(self.state_handler.follow, (node_to_follow, self.cluster.leader, recovery))
|
||||
self._async_executor.run_async(self.state_handler.follow_the_leader, (leader, ))
|
||||
return ret
|
||||
|
||||
def enforce_master_role(self, message, promote_message):
|
||||
@@ -165,8 +190,9 @@ class Ha(object):
|
||||
logger.info('Got response from %s %s: %s', member.name, member.api_url, response.content)
|
||||
json = response.json()
|
||||
is_master = json['role'] == 'master'
|
||||
xlog_location = None if is_master else json['xlog']['replayed_location']
|
||||
return (member, True, not is_master, xlog_location, json.get('tags', {}))
|
||||
xlog_location = json['xlog']['location' if is_master else 'replayed_location']
|
||||
tags = json.get('tags', dict())
|
||||
return (member, True, not is_master, xlog_location, tags)
|
||||
except:
|
||||
logging.exception('request failed: GET %s', member.api_url)
|
||||
return (member, False, None, 0, {})
|
||||
@@ -181,6 +207,12 @@ class Ha(object):
|
||||
def _is_healthiest_node(self, members, check_replication_lag=True):
|
||||
"""This method tries to determine whether I am healthy enough to became a new leader candidate or not."""
|
||||
|
||||
if self.state_handler.is_leader():
|
||||
return True
|
||||
|
||||
if self.patroni.nofailover is True:
|
||||
return False
|
||||
|
||||
if check_replication_lag and not self.state_handler.check_replication_lag(self.cluster.last_leader_operation):
|
||||
return False # Too far behind last reported xlog location on master
|
||||
|
||||
@@ -202,7 +234,7 @@ class Ha(object):
|
||||
ret = False
|
||||
members = [m for m in members if m.name != self.state_handler.name and not m.nofailover and m.api_url]
|
||||
if members:
|
||||
for member, reachable, _, _, tags in self.fetch_nodes_statuses(members):
|
||||
for member, reachable, in_recovery, xlog_location, tags in self.fetch_nodes_statuses(members):
|
||||
if reachable and not tags.get('nofailover', False):
|
||||
ret = True # TODO: check xlog_location
|
||||
elif not reachable:
|
||||
@@ -215,14 +247,14 @@ class Ha(object):
|
||||
|
||||
def manual_failover_process_no_leader(self):
|
||||
failover = self.cluster.failover
|
||||
if failover.candidate: # manual failover to specific member
|
||||
if failover.candidate == self.state_handler.name: # manual failover to me
|
||||
if failover.member: # manual failover to specific member
|
||||
if failover.member == self.state_handler.name: # manual failover to me
|
||||
return True
|
||||
|
||||
# find specific node and check that it is healthy
|
||||
member = self.cluster.get_member(failover.candidate, fallback_to_leader=False)
|
||||
if member:
|
||||
member, reachable, _, _, tags = self.fetch_node_status(member)
|
||||
members = [m for m in self.cluster.members if m.name == failover.member]
|
||||
if members:
|
||||
member, reachable, in_recovery, xlog_location, tags = self.fetch_node_status(members[0])
|
||||
if reachable and not tags.get('nofailover', False): # node is healthy
|
||||
logger.info('manual failover: to %s, i am %s', member.name, self.state_handler.name)
|
||||
return False
|
||||
@@ -233,13 +265,13 @@ class Ha(object):
|
||||
logger.warning('manual failover: member %s is not allowed to promote', member.name)
|
||||
|
||||
# at this point we should consider all members as a candidates for failover
|
||||
# i.e. we assume that failover.candidate is None
|
||||
# i.e. we assume that failover.member is None
|
||||
|
||||
# try to pick some other members to failover and check that they are healthy
|
||||
if failover.leader:
|
||||
if self.state_handler.name == failover.leader: # I was the leader
|
||||
# exclude me and desired member which is unhealthy (failover.candidate can be None)
|
||||
members = [m for m in self.cluster.members if m.name not in (failover.candidate, failover.leader)]
|
||||
# exclude me and desired member which is unhealthy (failover.member can be None)
|
||||
members = [m for m in self.cluster.members if m.name not in (failover.member, failover.leader)]
|
||||
if self.is_failover_possible(members): # check that there are healthy members
|
||||
return False
|
||||
else: # I was the leader and it looks like currently I am the only healthy member
|
||||
@@ -252,6 +284,7 @@ class Ha(object):
|
||||
return self._is_healthiest_node(members, check_replication_lag=False)
|
||||
|
||||
def is_healthiest_node(self):
|
||||
|
||||
if self.state_handler.is_leader(): # leader is always the healthiest
|
||||
return True
|
||||
|
||||
@@ -268,49 +301,23 @@ class Ha(object):
|
||||
def demote(self, delete_leader=True):
|
||||
if delete_leader:
|
||||
self.state_handler.stop()
|
||||
self.state_handler.set_role('unknown')
|
||||
self.dcs.delete_leader()
|
||||
self.touch_member()
|
||||
self.dcs.reset_cluster()
|
||||
sleep(2) # Give a time to somebody to promote
|
||||
cluster = self.dcs.get_cluster()
|
||||
node_to_follow = self._get_node_to_follow(cluster)
|
||||
self.state_handler.follow(node_to_follow, cluster.leader, True)
|
||||
else:
|
||||
self.state_handler.follow(None, None)
|
||||
self.state_handler.follow_the_leader(None)
|
||||
|
||||
def process_manual_failover_from_leader(self):
|
||||
def process_manual_failover_from_leader(self, bdr=False):
|
||||
failover = self.cluster.failover
|
||||
|
||||
if failover.scheduled_at:
|
||||
# If the failover is in the far future, we shouldn't do anything and just return.
|
||||
# If the failover is in the past, we consider the value to be stale and we remove
|
||||
# the value.
|
||||
# If the value is close to now, we initiate the failover
|
||||
now = datetime.datetime.now(pytz.utc)
|
||||
try:
|
||||
delta = (failover.scheduled_at - now).total_seconds()
|
||||
|
||||
if delta > self.patroni.nap_time:
|
||||
logging.info('Awaiting failover at %s (in %.0f seconds)', failover.scheduled_at.isoformat(), delta)
|
||||
return
|
||||
elif delta < - int(self.patroni.nap_time * 1.5):
|
||||
logger.warning('Found a stale failover value, cleaning up: %s', failover.scheduled_at)
|
||||
self.dcs.manual_failover('', '', index=self.cluster.failover.index)
|
||||
return
|
||||
|
||||
# The value is very close to now
|
||||
sleep(max(delta, 0))
|
||||
logger.info('Manual scheduled failover at {}'.format(failover.scheduled_at.isoformat()))
|
||||
except TypeError:
|
||||
logger.warning('Incorrect value in of scheduled_at: %s', failover.scheduled_at)
|
||||
|
||||
if not failover.leader or failover.leader == self.state_handler.name:
|
||||
if not failover.candidate or failover.candidate != self.state_handler.name:
|
||||
members = [m for m in self.cluster.members if not failover.candidate or m.name == failover.candidate]
|
||||
if not failover.member or failover.member != self.state_handler.name:
|
||||
members = [m for m in self.cluster.members if not failover.member or m.name == failover.member]
|
||||
if self.is_failover_possible(members): # check that there are healthy members
|
||||
self._async_executor.schedule('manual failover: demote')
|
||||
self._async_executor.run_async(self.demote)
|
||||
if not bdr:
|
||||
self._async_executor.schedule('manual failover: demote')
|
||||
self._async_executor.run_async(self.demote)
|
||||
else:
|
||||
self.dcs.delete_leader()
|
||||
self.dcs.reset_cluster()
|
||||
return 'manual failover: demoting myself'
|
||||
else:
|
||||
logger.warning('manual failover: no healthy members found, failover is not possible')
|
||||
@@ -321,7 +328,10 @@ class Ha(object):
|
||||
self.cluster.failover.leader, self.state_handler.name)
|
||||
|
||||
logger.info('Trying to clean up failover key')
|
||||
self.dcs.manual_failover('', '', index=self.cluster.failover.index)
|
||||
self.dcs.manual_failover('', '', self.cluster.failover.index)
|
||||
|
||||
def process_manual_failover_from_leader_bdr(self):
|
||||
self.process_manual_failover_from_leader(bdr=True)
|
||||
|
||||
def process_unhealthy_cluster(self):
|
||||
if self.is_healthiest_node():
|
||||
@@ -329,18 +339,18 @@ class Ha(object):
|
||||
if self.cluster.failover:
|
||||
logger.info('Cleaning up failover key after acquiring leader lock...')
|
||||
self.dcs.manual_failover('', '')
|
||||
self.load_cluster_from_dcs()
|
||||
self.dcs.get_cluster()
|
||||
return self.enforce_master_role('acquired session lock as a leader',
|
||||
'promoted self to leader by acquiring session lock')
|
||||
else:
|
||||
return self.follow('demoted self after trying and failing to obtain lock',
|
||||
'following new leader after trying and failing to obtain lock')
|
||||
return self.follow_the_leader('demoted self due after trying and failing to obtain lock',
|
||||
'following new leader after trying and failing to obtain lock')
|
||||
else:
|
||||
if self.patroni.nofailover:
|
||||
return self.follow('demoting self because I am not allowed to become master',
|
||||
'following a different leader because I am not allowed to promote')
|
||||
return self.follow('demoting self because i am not the healthiest node',
|
||||
'following a different leader because i am not the healthiest node')
|
||||
return self.follow_the_leader('demoting self because I am not allowed to become master',
|
||||
'following a different leader because I am not allowed to promote')
|
||||
return self.follow_the_leader('demoting self because i am not the healthiest node',
|
||||
'following a different leader because i am not the healthiest node')
|
||||
|
||||
def process_healthy_cluster(self):
|
||||
if self.has_lock():
|
||||
@@ -358,8 +368,33 @@ class Ha(object):
|
||||
self.load_cluster_from_dcs()
|
||||
else:
|
||||
logger.info('does not have lock')
|
||||
return self.follow('demoting self because i do not have the lock and i was a leader',
|
||||
'no action. i am a secondary and i am following a leader', False)
|
||||
return self.follow_the_leader('demoting self because i do not have the lock and i was a leader',
|
||||
'no action. i am a secondary and i am following a leader', False)
|
||||
|
||||
def process_unhealthy_cluster_bdr(self):
|
||||
if self.acquire_lock():
|
||||
if self.cluster.failover:
|
||||
logger.info('Cleaning up failover key after acquiring leader lock...')
|
||||
self.dcs.manual_failover('', '')
|
||||
self.dcs.get_cluster()
|
||||
return "acquired the session lock as a leader"
|
||||
else:
|
||||
return "lost the race to acquire the session lock"
|
||||
|
||||
def process_healthy_cluster_bdr(self):
|
||||
if self.has_lock():
|
||||
if self.cluster.failover:
|
||||
msg = self.process_manual_failover_from_leader_bdr()
|
||||
if msg is not None:
|
||||
return msg
|
||||
if self.update_lock():
|
||||
return "no action. I am the leader with the lock"
|
||||
else:
|
||||
logger.error("failed to update leader lock")
|
||||
self.load_cluster_from_dcs()
|
||||
else:
|
||||
logger.info("does not have lock")
|
||||
return "no action. I do not have the lock"
|
||||
|
||||
def schedule(self, action):
|
||||
with self._async_executor:
|
||||
@@ -387,10 +422,7 @@ class Ha(object):
|
||||
def reinitialize(self, cluster):
|
||||
self.state_handler.stop('immediate')
|
||||
self.state_handler.remove_data_directory()
|
||||
|
||||
clone_member = cluster.get_clone_member()
|
||||
member_role = 'leader' if clone_member == cluster.leader else 'replica'
|
||||
self.clone(clone_member, "from {0} '{1}'".format(member_role, clone_member.name))
|
||||
self.copy_backup_from_leader(cluster.leader)
|
||||
|
||||
def process_scheduled_action(self):
|
||||
if self.reinitialize_scheduled():
|
||||
@@ -415,45 +447,31 @@ class Ha(object):
|
||||
else:
|
||||
return self._async_executor.scheduled_action + ' in progress'
|
||||
|
||||
@staticmethod
|
||||
def sysid_valid(sysid):
|
||||
def sysid_valid(self, sysid):
|
||||
# sysid does tv_sec << 32, where tv_sec is the number of seconds sine 1970,
|
||||
# so even 1 << 32 would have 10 digits.
|
||||
sysid = str(sysid)
|
||||
return len(sysid) >= 10 and sysid.isdigit()
|
||||
|
||||
def post_recover(self):
|
||||
if not self.state_handler.is_running():
|
||||
if self.has_lock():
|
||||
self.dcs.delete_leader()
|
||||
self.dcs.reset_cluster()
|
||||
return 'removed leader key after trying and failing to start postgres'
|
||||
return 'failed to start postgres'
|
||||
return None
|
||||
return str(sysid) and len(str(sysid)) >= 10 and str(sysid).isdigit()
|
||||
|
||||
def _run_cycle(self):
|
||||
try:
|
||||
self.load_cluster_from_dcs()
|
||||
|
||||
self.touch_member()
|
||||
# if member key is missing - we may need to destory
|
||||
# and re-create the BDR database unless we are the first node in the cluster
|
||||
if self.use_bdr and len([m for m in self.cluster.members if m.name == self.state_handler.name]) == 0:
|
||||
self.bdr_may_need_reinitialize = True
|
||||
|
||||
# Create a member key with a very short expiration time if we may need to reinitialize this member
|
||||
# in order to avoid lossing the "may need to reinitialize status" in case Patroni is restarted
|
||||
self.touch_member(ttl=None if not self.bdr_may_need_reinitialize else self.bdr.get('min_ttl', 5))
|
||||
|
||||
# cluster has leader key but not initialize key
|
||||
if not (self.cluster.is_unlocked() or self.sysid_valid(self.cluster.initialize)) and self.has_lock():
|
||||
if not self.cluster.is_unlocked() and not self.sysid_valid(self.cluster.initialize) and self.has_lock():
|
||||
self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=self.state_handler.sysid)
|
||||
|
||||
if not (self.cluster.is_unlocked() or self.cluster.config and self.cluster.config.data) and self.has_lock():
|
||||
self.dcs.set_config_value(json.dumps(self.patroni.config.dynamic_configuration, separators=(',', ':')))
|
||||
|
||||
if self._async_executor.busy:
|
||||
return self.handle_long_action_in_progress()
|
||||
|
||||
# we've got here, so any async action has finished. Check if we tried to recover and failed
|
||||
if self.recovering:
|
||||
self.recovering = False
|
||||
msg = self.post_recover()
|
||||
if msg is not None:
|
||||
return msg
|
||||
|
||||
# currently it can trigger only reinitialize
|
||||
msg = self.process_scheduled_action()
|
||||
if msg is not None:
|
||||
@@ -461,41 +479,50 @@ class Ha(object):
|
||||
|
||||
# is data directory empty?
|
||||
if self.state_handler.data_directory_empty():
|
||||
return self.bootstrap() # new node
|
||||
if self.use_bdr:
|
||||
# member key was missing, but it's a new database - no need to reinitialize
|
||||
self.bdr_may_need_reinitialize = False
|
||||
self.touch_member()
|
||||
return self.bootstrap(self.use_bdr) # new node
|
||||
# "bootstrap", but data directory is not empty
|
||||
elif not self.sysid_valid(self.cluster.initialize) and self.cluster.is_unlocked():
|
||||
elif not self.sysid_valid(self.cluster.initialize) and self.cluster.is_unlocked() and \
|
||||
not self.bdr_may_need_reinitialize:
|
||||
self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=self.state_handler.sysid)
|
||||
else:
|
||||
# check if we are allowed to join
|
||||
if self.sysid_valid(self.cluster.initialize) and self.cluster.initialize != self.state_handler.sysid:
|
||||
logger.fatal("system ID mismatch, node %s belongs to a different cluster: %s != %s",
|
||||
self.state_handler.name, self.cluster.initialize, self.state_handler.sysid)
|
||||
if not self.use_bdr and \
|
||||
self.sysid_valid(self.cluster.initialize) and self.cluster.initialize != self.state_handler.sysid:
|
||||
logger.fatal("system ID mismatch, node {0} belongs to a different cluster".
|
||||
format(self.state_handler.name))
|
||||
sys.exit(1)
|
||||
|
||||
# try to start dead postgres
|
||||
if not self.state_handler.is_healthy():
|
||||
msg = self.recover()
|
||||
if msg is not None:
|
||||
return msg
|
||||
if self.use_bdr:
|
||||
if self.bdr_may_need_reinitialize or not self.state_handler.is_healthy():
|
||||
msg = self.recover_bdr()
|
||||
if msg is not None:
|
||||
return msg
|
||||
else:
|
||||
if not self.state_handler.is_healthy():
|
||||
msg = self.recover()
|
||||
if msg is not None:
|
||||
return msg
|
||||
|
||||
try:
|
||||
if self.cluster.is_unlocked():
|
||||
return self.process_unhealthy_cluster()
|
||||
return self.process_unhealthy_cluster() if not self.use_bdr else self.process_unhealthy_cluster_bdr()
|
||||
else:
|
||||
return self.process_healthy_cluster()
|
||||
return self.process_healthy_cluster() if not self.use_bdr else self.process_healthy_cluster_bdr()
|
||||
finally:
|
||||
# we might not have a valid PostgreSQL connection here if another thread
|
||||
# stops PostgreSQL, therefore, we only reload replication slots if no
|
||||
# asynchronous processes are running (should be always the case for the master)
|
||||
if not self._async_executor.busy:
|
||||
if not self.use_bdr:
|
||||
self.state_handler.sync_replication_slots(self.cluster)
|
||||
except DCSError:
|
||||
logger.error('Error communicating with DCS')
|
||||
if self.state_handler.is_running() and self.state_handler.is_leader():
|
||||
if self.state_handler.is_running() and self.state_handler.is_leader() and not self.use_bdr:
|
||||
self.demote(delete_leader=False)
|
||||
return 'demoted self because DCS is not accessible and i was a leader'
|
||||
except (psycopg2.Error, PostgresConnectionException):
|
||||
logger.exception('Error communicating with PostgreSQL. Will try again later')
|
||||
logger.exception('Error communicating with Postgresql. Will try again later')
|
||||
|
||||
def run_cycle(self):
|
||||
with self._async_executor:
|
||||
|
||||
+398
-535
File diff suppressed because it is too large
Load Diff
@@ -9,7 +9,7 @@ import boto.ec2
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AWSConnection(object):
|
||||
class AWSConnection:
|
||||
def __init__(self, cluster_name):
|
||||
self.available = False
|
||||
self.cluster_name = cluster_name if cluster_name is not None else 'unknown'
|
||||
@@ -56,7 +56,7 @@ class AWSConnection(object):
|
||||
conn = boto.ec2.connect_to_region(self.region)
|
||||
conn.create_tags([self.instance_id], tags)
|
||||
except Exception as e:
|
||||
logger.info("could not set tags for EC2 instance %s: %s", self.instance_id, e)
|
||||
logger.info("could not set tags for EC2 instance {}: {}".format(self.instance_id, e))
|
||||
return False
|
||||
return True
|
||||
|
||||
@@ -66,7 +66,6 @@ class AWSConnection(object):
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
|
||||
if len(sys.argv) == 4 and sys.argv[1] in ('on_start', 'on_stop', 'on_role_change'):
|
||||
AWSConnection(cluster_name=sys.argv[3]).on_role_change(sys.argv[2])
|
||||
else:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#!/usr/bin/env python
|
||||
#!/usr/bin/python
|
||||
|
||||
# sample script to clone new replicas using WAL-E restore
|
||||
# falls back to pg_basebackup if WAL-E restore fails, or if
|
||||
@@ -33,15 +33,16 @@ import sys
|
||||
import argparse
|
||||
|
||||
|
||||
if sys.hexversion >= 0x0300000:
|
||||
if sys.hexversion >= 0x03000000:
|
||||
long = int
|
||||
|
||||
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WALERestore(object):
|
||||
|
||||
def __init__(self, scope, datadir, connstring, env_dir, threshold_mb, threshold_pct, use_iam, no_master):
|
||||
def __init__(self, scope, datadir, connstring, env_dir, threshold_mb, threshold_pct, use_iam):
|
||||
self.scope = scope
|
||||
self.master_connection = connstring
|
||||
self.data_dir = datadir
|
||||
@@ -50,7 +51,6 @@ class WALERestore(object):
|
||||
self.wal_e.threshold_mb = threshold_mb
|
||||
self.wal_e.threshold_pct = threshold_pct
|
||||
self.wal_e.iam_string = ' --aws-instance-profile ' if use_iam == 1 else ''
|
||||
self.no_master = no_master
|
||||
self.wal_e.cmd = 'envdir {0} wal-e {1} '.format(self.wal_e.dir, self.wal_e.iam_string)
|
||||
self.init_error = (not os.path.exists(self.wal_e.dir))
|
||||
|
||||
@@ -104,23 +104,24 @@ class WALERestore(object):
|
||||
lsn_offset = hex((long(backup_start_segment[16:32], 16) << 24) + long(backup_start_offset))[2:-1]
|
||||
|
||||
# construct the LSN from the segment and offset
|
||||
backup_start_lsn = '{0}/{1}'.format(lsn_segment, lsn_offset)
|
||||
backup_start_lsn = '{}/{}'.format(lsn_segment, lsn_offset)
|
||||
|
||||
conn = None
|
||||
cursor = None
|
||||
diff_in_bytes = long(backup_size)
|
||||
if not self.no_master:
|
||||
try:
|
||||
# get the difference in bytes between the current WAL location and the backup start offset
|
||||
with psycopg2.connect(self.master_connection) as con:
|
||||
con.autocommit = True
|
||||
with con.cursor() as cur:
|
||||
cur.execute("SELECT pg_xlog_location_diff(pg_current_xlog_location(), %s)", (backup_start_lsn,))
|
||||
diff_in_bytes = long(cur.fetchone()[0])
|
||||
except psycopg2.Error as e:
|
||||
logger.error('could not determine difference with the master location: %s', e)
|
||||
return False
|
||||
else:
|
||||
# always try to use WAL-E if base backup is available
|
||||
diff_in_bytes = 0
|
||||
try:
|
||||
# get the difference in bytes between the current WAL location and the backup start offset
|
||||
conn = psycopg2.connect(self.master_connection)
|
||||
conn.autocommit = True
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT pg_xlog_location_diff(pg_current_xlog_location(), %s)", (backup_start_lsn,))
|
||||
diff_in_bytes = long(cursor.fetchone()[0])
|
||||
except psycopg2.Error as e:
|
||||
logger.error('could not determine difference with the master location: {}'.format(e))
|
||||
return False
|
||||
finally:
|
||||
cursor and cursor.close()
|
||||
conn and conn.close()
|
||||
|
||||
# if the size of the accumulated WAL segments is more than a certan percentage of the backup size
|
||||
# or exceeds the pre-determined size - pg_basebackup is chosen instead.
|
||||
@@ -139,7 +140,6 @@ class WALERestore(object):
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
|
||||
parser = argparse.ArgumentParser(description='Script to image replicas using WAL-E')
|
||||
parser.add_argument('--scope', required=True)
|
||||
parser.add_argument('--role', required=False)
|
||||
@@ -150,15 +150,13 @@ def main():
|
||||
parser.add_argument('--threshold_megabytes', type=int, default=10240)
|
||||
parser.add_argument('--threshold_backup_size_percentage', type=int, default=30)
|
||||
parser.add_argument('--use_iam', type=int, default=0)
|
||||
parser.add_argument('--no_master', type=int, default=0)
|
||||
args = parser.parse_args()
|
||||
|
||||
# retry cloning in a loop
|
||||
for _ in range(0, args.retries + 1):
|
||||
for retry in range(0, args.retries + 1):
|
||||
restore = WALERestore(scope=args.scope, datadir=args.datadir, connstring=args.connstring,
|
||||
env_dir=args.envdir, threshold_mb=args.threshold_megabytes,
|
||||
threshold_pct=args.threshold_backup_size_percentage, use_iam=args.use_iam,
|
||||
no_master=args.no_master)
|
||||
threshold_pct=args.threshold_backup_size_percentage, use_iam=args.use_iam)
|
||||
ret = restore.run()
|
||||
if ret == 0:
|
||||
break
|
||||
|
||||
+55
-189
@@ -1,220 +1,86 @@
|
||||
import datetime
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
|
||||
from patroni.exceptions import PatroniException
|
||||
|
||||
if sys.hexversion >= 0x0300000:
|
||||
long = int
|
||||
ignore_sigterm = False
|
||||
interrupted_sleep = False
|
||||
reap_children = False
|
||||
|
||||
__interrupted_sleep = False
|
||||
__reap_children = False
|
||||
_DATE_TIME_RE = re.compile(r'''^
|
||||
(?P<year>\d{4})\-(?P<month>\d{2})\-(?P<day>\d{2}) # date
|
||||
T
|
||||
(?P<hour>\d{2}):(?P<minute>\d{2}):(?P<second>\d{2})\.(?P<microsecond>\d{6}) # time
|
||||
\d*Z$''', re.X)
|
||||
|
||||
|
||||
def deep_compare(obj1, obj2):
|
||||
def parse_datetime(time_str):
|
||||
"""
|
||||
>>> deep_compare({'1': None}, {})
|
||||
False
|
||||
>>> deep_compare({'1': {}}, {'1': None})
|
||||
False
|
||||
>>> deep_compare({'1': [1]}, {'1': [2]})
|
||||
False
|
||||
>>> deep_compare({'1': 2}, {'1': '2'})
|
||||
True
|
||||
>>> deep_compare({'1': {'2': [3, 4]}}, {'1': {'2': [3, 4]}})
|
||||
>>> parse_datetime('2015-06-10T12:56:30.552539016Z')
|
||||
datetime.datetime(2015, 6, 10, 12, 56, 30, 552539)
|
||||
>>> parse_datetime('2015-06-10 12:56:30.552539016Z')
|
||||
"""
|
||||
m = _DATE_TIME_RE.match(time_str)
|
||||
if not m:
|
||||
return None
|
||||
p = dict((n, int(m.group(n))) for n in 'year month day hour minute second microsecond'.split(' '))
|
||||
return datetime.datetime(**p)
|
||||
|
||||
|
||||
def calculate_ttl(expiration):
|
||||
"""
|
||||
>>> calculate_ttl(None)
|
||||
>>> calculate_ttl('2015-06-10 12:56:30.552539016Z')
|
||||
>>> calculate_ttl('2015-06-10T12:56:30.552539016Z') < 0
|
||||
True
|
||||
"""
|
||||
|
||||
if set(list(obj1.keys())) != set(list(obj2.keys())): # Objects have different sets of keys
|
||||
return False
|
||||
|
||||
for key, value in obj1.items():
|
||||
if isinstance(value, dict):
|
||||
if not (isinstance(obj2[key], dict) and deep_compare(value, obj2[key])):
|
||||
return False
|
||||
elif str(value) != str(obj2[key]):
|
||||
return False
|
||||
return True
|
||||
if not expiration:
|
||||
return None
|
||||
expiration = parse_datetime(expiration)
|
||||
if not expiration:
|
||||
return None
|
||||
now = datetime.datetime.utcnow()
|
||||
return int((expiration - now).total_seconds())
|
||||
|
||||
|
||||
def patch_config(config, data):
|
||||
"""recursively 'patch' `config` with `data`
|
||||
:returns: `!True` if the `config` was changed"""
|
||||
is_changed = False
|
||||
for name, value in data.items():
|
||||
if value is None:
|
||||
if config.pop(name, None) is not None:
|
||||
is_changed = True
|
||||
elif name in config:
|
||||
if isinstance(value, dict):
|
||||
if isinstance(config[name], dict):
|
||||
if patch_config(config[name], value):
|
||||
is_changed = True
|
||||
else:
|
||||
config[name] = value
|
||||
is_changed = True
|
||||
elif str(config[name]) != str(value):
|
||||
config[name] = value
|
||||
is_changed = True
|
||||
else:
|
||||
config[name] = value
|
||||
is_changed = True
|
||||
return is_changed
|
||||
|
||||
|
||||
def parse_bool(value):
|
||||
"""
|
||||
>>> parse_bool(1)
|
||||
True
|
||||
>>> parse_bool('off')
|
||||
False
|
||||
>>> parse_bool('foo')
|
||||
"""
|
||||
value = str(value).lower()
|
||||
if value in ('on', 'true', 'yes', '1'):
|
||||
return True
|
||||
if value in ('off', 'false', 'no', '0'):
|
||||
return False
|
||||
|
||||
|
||||
def strtol(value, strict=True):
|
||||
"""As most as possible close equivalent of strtol(3) function (with base=0),
|
||||
used by postgres to parse parameter values.
|
||||
>>> strtol(0) == (0, '')
|
||||
True
|
||||
>>> strtol(1) == (1, '')
|
||||
True
|
||||
>>> strtol(9) == (9, '')
|
||||
True
|
||||
>>> strtol(' +0x400MB') == (1024, 'MB')
|
||||
True
|
||||
>>> strtol(' -070d') == (-56, 'd')
|
||||
True
|
||||
>>> strtol(' d ') == (None, 'd')
|
||||
True
|
||||
>>> strtol('9s', False) == (9, 's')
|
||||
True
|
||||
>>> strtol(' s ', False) == (1, 's')
|
||||
True
|
||||
"""
|
||||
value = str(value).strip()
|
||||
l = len(value)
|
||||
i = 0
|
||||
# skip sign:
|
||||
if i < l and value[i] in ('-', '+'):
|
||||
i += 1
|
||||
|
||||
# we always expect to get digit in the beginning
|
||||
if i < l and value[i].isdigit():
|
||||
if value[i] == '0':
|
||||
i += 1
|
||||
if i < l and value[i] in ('x', 'X'): # '0' followed by 'x': HEX
|
||||
base = 16
|
||||
i += 1
|
||||
else: # just starts with '0': OCT
|
||||
base = 8
|
||||
else: # any other digit: DEC
|
||||
base = 10
|
||||
|
||||
ret = None
|
||||
while i <= l:
|
||||
try: # try to find maximally long number
|
||||
i += 1 # by giving to `int` longer and longer strings
|
||||
ret = long(value[:i], base)
|
||||
except ValueError: # until we will not get an exception or end of the string
|
||||
i -= 1
|
||||
break
|
||||
if ret is not None: # yay! there is a number in the beginning of the string
|
||||
return ret, value[i:].strip() # return the number and the "rest"
|
||||
|
||||
return (None if strict else 1), value.strip()
|
||||
|
||||
|
||||
def parse_int(value, base_unit=None):
|
||||
"""
|
||||
>>> parse_int('1') == 1
|
||||
True
|
||||
>>> parse_int(' 0x400 MB ', '16384kB') == 64
|
||||
True
|
||||
>>> parse_int('1MB', 'kB') == 1024
|
||||
True
|
||||
>>> parse_int('1000 ms', 's') == 1
|
||||
True
|
||||
>>> parse_int('1GB', 'MB') is None
|
||||
True
|
||||
>>> parse_int(0) == 0
|
||||
True
|
||||
"""
|
||||
|
||||
convert = {
|
||||
'kB': {'kB': 1, 'MB': 1024, 'GB': 1024 * 1024, 'TB': 1024 * 1024 * 1024},
|
||||
'ms': {'ms': 1, 's': 1000, 'min': 1000 * 60, 'h': 1000 * 60 * 60, 'd': 1000 * 60 * 60 * 24},
|
||||
's': {'ms': -1000, 's': 1, 'min': 60, 'h': 60 * 60, 'd': 60 * 60 * 24},
|
||||
'min': {'ms': -1000 * 60, 's': -60, 'min': 1, 'h': 60, 'd': 60 * 24}
|
||||
}
|
||||
|
||||
value, unit = strtol(value)
|
||||
if value is not None:
|
||||
if not unit:
|
||||
return value
|
||||
|
||||
if base_unit and base_unit not in convert:
|
||||
base_value, base_unit = strtol(base_unit, False)
|
||||
else:
|
||||
base_value = 1
|
||||
if base_unit in convert and unit in convert[base_unit]:
|
||||
multiplier = convert[base_unit][unit]
|
||||
if multiplier < 0:
|
||||
value /= -multiplier
|
||||
else:
|
||||
value *= multiplier
|
||||
return int(value/base_value)
|
||||
|
||||
|
||||
def compare_values(vartype, unit, old_value, new_value):
|
||||
"""
|
||||
>>> compare_values('enum', None, 'remote_write', 'REMOTE_WRITE')
|
||||
True
|
||||
>>> compare_values('real', None, '1.23', 1.23)
|
||||
True
|
||||
"""
|
||||
|
||||
# if the integer or bool new_value is not correct this function will return False
|
||||
if vartype == 'bool':
|
||||
old_value = parse_bool(old_value)
|
||||
new_value = parse_bool(new_value)
|
||||
elif vartype == 'integer':
|
||||
old_value = parse_int(old_value)
|
||||
new_value = parse_int(new_value, unit)
|
||||
elif vartype == 'enum':
|
||||
return str(old_value).lower() == str(new_value).lower()
|
||||
else: # ('string', 'real')
|
||||
return str(old_value) == str(new_value)
|
||||
return old_value is not None and new_value is not None and old_value == new_value
|
||||
def sigterm_handler(signo, stack_frame):
|
||||
global ignore_sigterm
|
||||
if not ignore_sigterm:
|
||||
ignore_sigterm = True
|
||||
sys.exit()
|
||||
|
||||
|
||||
def sigchld_handler(signo, stack_frame):
|
||||
global __interrupted_sleep, __reap_children
|
||||
__reap_children = __interrupted_sleep = True
|
||||
global interrupted_sleep, reap_children
|
||||
reap_children = interrupted_sleep = True
|
||||
|
||||
|
||||
def sleep(interval):
|
||||
global __interrupted_sleep
|
||||
global interrupted_sleep
|
||||
current_time = time.time()
|
||||
end_time = current_time + interval
|
||||
while current_time < end_time:
|
||||
__interrupted_sleep = False
|
||||
interrupted_sleep = False
|
||||
time.sleep(end_time - current_time)
|
||||
if not __interrupted_sleep: # we will ignore only sigchld
|
||||
if not interrupted_sleep: # we will ignore only sigchld
|
||||
break
|
||||
current_time = time.time()
|
||||
__interrupted_sleep = False
|
||||
interrupted_sleep = False
|
||||
|
||||
|
||||
def setup_signal_handlers():
|
||||
signal.signal(signal.SIGTERM, sigterm_handler)
|
||||
signal.signal(signal.SIGCHLD, sigchld_handler)
|
||||
|
||||
|
||||
def reap_children():
|
||||
global __reap_children
|
||||
if __reap_children:
|
||||
global reap_children
|
||||
if reap_children:
|
||||
try:
|
||||
while True:
|
||||
ret = os.waitpid(-1, os.WNOHANG)
|
||||
@@ -223,7 +89,7 @@ def reap_children():
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
__reap_children = False
|
||||
reap_children = False
|
||||
|
||||
|
||||
class RetryFailedError(PatroniException):
|
||||
@@ -231,7 +97,7 @@ class RetryFailedError(PatroniException):
|
||||
"""Raised when retrying an operation ultimately failed, after retrying the maximum number of attempts."""
|
||||
|
||||
|
||||
class Retry(object):
|
||||
class Retry:
|
||||
|
||||
"""Helper for retrying a method in the face of retry-able exceptions"""
|
||||
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
__version__ = '1.0'
|
||||
__version__ = '0.76'
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
import logging
|
||||
import random
|
||||
import requests
|
||||
import time
|
||||
|
||||
from kazoo.client import KazooClient, KazooState
|
||||
from kazoo.exceptions import NoNodeError, NodeExistsError
|
||||
from patroni.dcs import AbstractDCS, Cluster, Failover, Leader, Member
|
||||
from patroni.exceptions import DCSError
|
||||
from patroni.utils import sleep
|
||||
from requests.exceptions import RequestException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ZooKeeperError(DCSError):
|
||||
pass
|
||||
|
||||
|
||||
class ExhibitorEnsembleProvider:
|
||||
|
||||
TIMEOUT = 3.1
|
||||
|
||||
def __init__(self, hosts, port, uri_path='/exhibitor/v1/cluster/list', poll_interval=300):
|
||||
self._exhibitor_port = port
|
||||
self._uri_path = uri_path
|
||||
self._poll_interval = poll_interval
|
||||
self._exhibitors = hosts
|
||||
self._master_exhibitors = hosts
|
||||
self._zookeeper_hosts = ''
|
||||
self._next_poll = None
|
||||
while not self.poll():
|
||||
logger.info('waiting on exhibitor')
|
||||
sleep(5)
|
||||
|
||||
def poll(self):
|
||||
if self._next_poll and self._next_poll > time.time():
|
||||
return False
|
||||
|
||||
json = self._query_exhibitors(self._exhibitors)
|
||||
if not json:
|
||||
json = self._query_exhibitors(self._master_exhibitors)
|
||||
|
||||
if isinstance(json, dict) and 'servers' in json and 'port' in json:
|
||||
self._next_poll = time.time() + self._poll_interval
|
||||
zookeeper_hosts = ','.join([h + ':' + str(json['port']) for h in sorted(json['servers'])])
|
||||
if self._zookeeper_hosts != zookeeper_hosts:
|
||||
logger.info('ZooKeeper connection string has changed: %s => %s', self._zookeeper_hosts, zookeeper_hosts)
|
||||
self._zookeeper_hosts = zookeeper_hosts
|
||||
self._exhibitors = json['servers']
|
||||
return True
|
||||
return False
|
||||
|
||||
def _query_exhibitors(self, exhibitors):
|
||||
random.shuffle(exhibitors)
|
||||
for host in exhibitors:
|
||||
uri = 'http://{}:{}{}'.format(host, self._exhibitor_port, self._uri_path)
|
||||
try:
|
||||
response = requests.get(uri, timeout=self.TIMEOUT)
|
||||
return response.json()
|
||||
except RequestException:
|
||||
pass
|
||||
return None
|
||||
|
||||
@property
|
||||
def zookeeper_hosts(self):
|
||||
return self._zookeeper_hosts
|
||||
|
||||
|
||||
class ZooKeeper(AbstractDCS):
|
||||
|
||||
def __init__(self, name, config):
|
||||
super(ZooKeeper, self).__init__(name, config)
|
||||
|
||||
hosts = config.get('hosts', [])
|
||||
if isinstance(hosts, list):
|
||||
hosts = ','.join(hosts)
|
||||
|
||||
self.exhibitor = None
|
||||
if 'exhibitor' in config:
|
||||
exhibitor = config['exhibitor']
|
||||
interval = exhibitor.get('poll_interval', 300)
|
||||
self.exhibitor = ExhibitorEnsembleProvider(exhibitor['hosts'], exhibitor['port'], poll_interval=interval)
|
||||
hosts = self.exhibitor.zookeeper_hosts
|
||||
|
||||
self.client = KazooClient(hosts=hosts,
|
||||
timeout=(config.get('session_timeout', None) or 30),
|
||||
command_retry={
|
||||
'deadline': (config.get('reconnect_timeout', None) or 10),
|
||||
'max_delay': 1,
|
||||
'max_tries': -1},
|
||||
connection_retry={'max_delay': 1, 'max_tries': -1})
|
||||
self.client.add_listener(self.session_listener)
|
||||
|
||||
self._my_member_data = None
|
||||
self.fetch_cluster = True
|
||||
self.last_leader_operation = 0
|
||||
|
||||
self.client.start(None)
|
||||
|
||||
def session_listener(self, state):
|
||||
if state in [KazooState.SUSPENDED, KazooState.LOST]:
|
||||
self.cluster_watcher(None)
|
||||
|
||||
def cluster_watcher(self, event):
|
||||
self.fetch_cluster = True
|
||||
self.event.set()
|
||||
|
||||
def get_node(self, key, watch=None):
|
||||
try:
|
||||
ret = self.client.get(key, watch)
|
||||
return (ret[0].decode('utf-8'), ret[1])
|
||||
except NoNodeError:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def member(name, value, znode):
|
||||
return Member.from_node(znode.version, name, znode.ephemeralOwner, value)
|
||||
|
||||
def get_children(self, key, watch=None):
|
||||
try:
|
||||
return self.client.get_children(key, watch)
|
||||
except NoNodeError:
|
||||
return []
|
||||
|
||||
def load_members(self):
|
||||
members = []
|
||||
for member in self.get_children(self.members_path, self.cluster_watcher):
|
||||
data = self.get_node(self.members_path + member)
|
||||
if data is not None:
|
||||
members.append(self.member(member, *data))
|
||||
return members
|
||||
|
||||
def _inner_load_cluster(self):
|
||||
self.fetch_cluster = False
|
||||
self.event.clear()
|
||||
nodes = set(self.get_children(self.client_path(''), self.cluster_watcher))
|
||||
if not nodes:
|
||||
self.fetch_cluster = True
|
||||
|
||||
# get initialize flag
|
||||
initialize = (self.get_node(self.initialize_path) or [None])[0] if self._INITIALIZE in nodes else None
|
||||
|
||||
# get list of members
|
||||
members = self.load_members() if self._MEMBERS[:-1] in nodes else []
|
||||
|
||||
# get leader
|
||||
leader = self.get_node(self.leader_path) if self._LEADER in nodes else None
|
||||
if leader:
|
||||
client_id = self.client.client_id
|
||||
if leader[0] == self._name and client_id is not None and client_id[0] != leader[1].ephemeralOwner:
|
||||
logger.info('I am leader but not owner of the session. Removing leader node')
|
||||
self.client.delete(self.leader_path)
|
||||
leader = None
|
||||
|
||||
if leader:
|
||||
member = Member(-1, leader[0], None, {})
|
||||
member = ([m for m in members if m.name == leader[0]] or [member])[0]
|
||||
leader = Leader(leader[1].version, leader[1].ephemeralOwner, member)
|
||||
self.fetch_cluster = member.index == -1
|
||||
|
||||
# failover key
|
||||
failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
|
||||
if failover:
|
||||
failover = Failover.from_node(failover[1].version, failover[0])
|
||||
|
||||
# get last leader operation
|
||||
optime = self.get_node(self.leader_optime_path) if self._OPTIME in nodes and self.fetch_cluster else None
|
||||
self.last_leader_operation = 0 if optime is None else int(optime[0])
|
||||
self._cluster = Cluster(initialize, leader, self.last_leader_operation, members, failover)
|
||||
|
||||
def _load_cluster(self):
|
||||
if self.exhibitor and self.exhibitor.poll():
|
||||
self.client.set_hosts(self.exhibitor.zookeeper_hosts)
|
||||
|
||||
if self.fetch_cluster:
|
||||
try:
|
||||
self.client.retry(self._inner_load_cluster)
|
||||
except:
|
||||
logger.exception('get_cluster')
|
||||
self.session_listener(KazooState.LOST)
|
||||
raise ZooKeeperError('ZooKeeper in not responding properly')
|
||||
|
||||
def _create(self, path, value, **kwargs):
|
||||
try:
|
||||
self.client.retry(self.client.create, path, value.encode('utf-8'), **kwargs)
|
||||
return True
|
||||
except:
|
||||
return False
|
||||
|
||||
def attempt_to_acquire_leader(self):
|
||||
ret = self._create(self.leader_path, self._name, makepath=True, ephemeral=True)
|
||||
ret or logger.info('Could not take out TTL lock')
|
||||
return ret
|
||||
|
||||
def set_failover_value(self, value, index=None):
|
||||
try:
|
||||
self.client.retry(self.client.set, self.failover_path, value.encode('utf-8'), version=index or -1)
|
||||
return True
|
||||
except NoNodeError:
|
||||
return value == '' or (not index and self._create(self.failover_path, value))
|
||||
except:
|
||||
logging.exception('set_failover_value')
|
||||
return False
|
||||
|
||||
def initialize(self, create_new=True, sysid=""):
|
||||
return self._create(self.initialize_path, sysid, makepath=True) if create_new \
|
||||
else self.client.retry(self.client.set, self.initialize_path, sysid.encode("utf-8"))
|
||||
|
||||
def touch_member(self, data, ttl=None):
|
||||
cluster = self.cluster
|
||||
me = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
|
||||
path = self.member_path
|
||||
data = data.encode('utf-8')
|
||||
create = not me
|
||||
if me and self.client.client_id is not None and me.session != self.client.client_id[0]:
|
||||
try:
|
||||
self.client.retry(self.client.delete, path)
|
||||
except NoNodeError:
|
||||
pass
|
||||
except:
|
||||
return False
|
||||
create = True
|
||||
|
||||
if not create and data == self._my_member_data:
|
||||
return True
|
||||
|
||||
try:
|
||||
if create:
|
||||
self.client.retry(self.client.create, path, data, makepath=True, ephemeral=True)
|
||||
else:
|
||||
self.client.retry(self.client.set, path, data)
|
||||
self._my_member_data = data
|
||||
return True
|
||||
except NodeExistsError:
|
||||
try:
|
||||
self.client.retry(self.client.set, path, data)
|
||||
self._my_member_data = data
|
||||
return True
|
||||
except:
|
||||
logger.exception('touch_member')
|
||||
except:
|
||||
logger.exception('touch_member')
|
||||
return False
|
||||
|
||||
def take_leader(self):
|
||||
return self.attempt_to_acquire_leader()
|
||||
|
||||
def write_leader_optime(self, last_operation):
|
||||
last_operation = last_operation.encode('utf-8')
|
||||
if last_operation != self.last_leader_operation:
|
||||
self.last_leader_operation = last_operation
|
||||
path = self.leader_optime_path
|
||||
try:
|
||||
self.client.retry(self.client.set, path, last_operation)
|
||||
except NoNodeError:
|
||||
try:
|
||||
self.client.retry(self.client.create, path, last_operation, makepath=True)
|
||||
except:
|
||||
logger.exception('Failed to create %s', path)
|
||||
except:
|
||||
logger.exception('Failed to update %s', path)
|
||||
|
||||
def update_leader(self):
|
||||
return True
|
||||
|
||||
def delete_leader(self):
|
||||
self.client.restart()
|
||||
self._my_member_data = None
|
||||
return True
|
||||
|
||||
def _cancel_initialization(self):
|
||||
node = self.get_node(self.initialize_path)
|
||||
if node:
|
||||
self.client.delete(self.initialize_path, version=node[1].version)
|
||||
|
||||
def cancel_initialization(self):
|
||||
try:
|
||||
self.client.retry(self._cancel_initialization)
|
||||
except:
|
||||
logger.exception("Unable to delete initialize key")
|
||||
|
||||
def watch(self, timeout):
|
||||
if super(ZooKeeper, self).watch(timeout):
|
||||
self.fetch_cluster = True
|
||||
return self.fetch_cluster
|
||||
+1
-1
@@ -2,4 +2,4 @@
|
||||
from patroni.ctl import ctl
|
||||
|
||||
if __name__ == '__main__':
|
||||
ctl(None)
|
||||
ctl()
|
||||
|
||||
Binary file not shown.
+95
-62
@@ -1,74 +1,107 @@
|
||||
scope: batman
|
||||
#namespace: /service/
|
||||
name: postgresql0
|
||||
|
||||
ttl: &ttl 30
|
||||
loop_wait: &loop_wait 10
|
||||
scope: &scope batman
|
||||
restapi:
|
||||
listen: 127.0.0.1:8008
|
||||
# authentication:
|
||||
# username: username
|
||||
# password: password
|
||||
connect_address: 127.0.0.1:8008
|
||||
|
||||
#auth: 'username:password'
|
||||
# certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
|
||||
# keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
|
||||
etcd:
|
||||
scope: *scope
|
||||
ttl: *ttl
|
||||
host: 127.0.0.1:4001
|
||||
|
||||
bootstrap:
|
||||
# this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
|
||||
# and all other cluster members will use it as a `global configuration`
|
||||
dcs:
|
||||
ttl: 30
|
||||
loop_wait: 10
|
||||
retry_timeout: 10
|
||||
maximum_lag_on_failover: 1048576
|
||||
postgresql:
|
||||
use_pg_rewind: true
|
||||
# use_slots: true
|
||||
parameters:
|
||||
# wal_level: hot_standby
|
||||
# hot_standby: "on"
|
||||
# wal_keep_segments: 8
|
||||
# max_wal_senders: 5
|
||||
# max_replication_slots: 5
|
||||
# wal_log_hints: "on"
|
||||
archive_mode: "on"
|
||||
archive_timeout: 1800s
|
||||
archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
|
||||
recovery_conf:
|
||||
restore_command: cp ../wal_archive/%f %p
|
||||
|
||||
# some desired options for 'initdb'
|
||||
initdb: # Note: It needs to be a list (some options need values, others are switches)
|
||||
- encoding: UTF8
|
||||
- data-checksums
|
||||
|
||||
pg_hba: # Add following lines to pg_hba.conf after running 'initdb'
|
||||
- host replication replicator 127.0.0.1/32 md5
|
||||
- host all all 0.0.0.0/0 md5
|
||||
# - hostssl all all 0.0.0.0/0 md5
|
||||
|
||||
# Some additional users users which needs to be created after initializing new cluster
|
||||
users:
|
||||
admin:
|
||||
password: admin
|
||||
options:
|
||||
- createrole
|
||||
- createdb
|
||||
|
||||
#discovery_srv: my-etcd.domain
|
||||
#zookeeper:
|
||||
# scope: *scope
|
||||
# session_timeout: *ttl
|
||||
# reconnect_timeout: *loop_wait
|
||||
# hosts:
|
||||
# - 127.0.0.1:2181
|
||||
# - 127.0.0.2:2181
|
||||
# exhibitor:
|
||||
# poll_interval: 300
|
||||
# port: 8181
|
||||
# hosts:
|
||||
# - host1
|
||||
# - host2
|
||||
# - host3
|
||||
bdr:
|
||||
enable: "on"
|
||||
database: "bdrtest"
|
||||
postgresql:
|
||||
name: postgresql0
|
||||
scope: *scope
|
||||
listen: 127.0.0.1:5432
|
||||
connect_address: 127.0.0.1:5432
|
||||
data_dir: data/postgresql0
|
||||
maximum_lag_on_failover: 1048576 # 1 megabyte in bytes
|
||||
use_slots: True
|
||||
pgpass: /tmp/pgpass0
|
||||
authentication:
|
||||
replication:
|
||||
username: replicator
|
||||
password: rep-pass
|
||||
superuser:
|
||||
username: postgres
|
||||
password: zalando
|
||||
initdb: ## We allow the following options to be passed on to initdb
|
||||
# - auth: authmethod
|
||||
# - auth-host: authmethod
|
||||
# - auth-local: authmethod
|
||||
- encoding: UTF8
|
||||
# - data-checksums # When pg_rewind is needed on 9.3, this needs to be enabled
|
||||
# - locale: locale
|
||||
# - lc-collate: locale
|
||||
# - lc-ctype: locale
|
||||
# - lc-messages: locale
|
||||
# - lc-monetary: locale
|
||||
# - lc-numeric: locale
|
||||
# - lc-time: locale
|
||||
# - text-search-config: CFG
|
||||
# - xlogdir: directory
|
||||
# - debug
|
||||
# - noclean
|
||||
pg_rewind:
|
||||
username: postgres
|
||||
password: zalando
|
||||
pg_hba:
|
||||
- host all all 0.0.0.0/0 md5
|
||||
- hostssl all all 0.0.0.0/0 md5
|
||||
replication:
|
||||
username: replicator
|
||||
password: rep-pass
|
||||
network: 127.0.0.1/32
|
||||
superuser:
|
||||
username: postgres
|
||||
password: zalando
|
||||
admin:
|
||||
username: admin
|
||||
password: admin
|
||||
create_replica_method:
|
||||
- basebackup
|
||||
# - wal_e
|
||||
# commented-out example for wal-e provisioning
|
||||
#create_replica_method: wal_e, basebackup
|
||||
#wal_e:
|
||||
#command: /patroni/scripts/wale_restore.py
|
||||
#env_dir: /etc/wal-e.d/env
|
||||
#threshold_megabytes: 10240
|
||||
#threshold_backup_size_percentage: 30
|
||||
#retries: 2
|
||||
#use_iam: 1
|
||||
#recovery_conf:
|
||||
#restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1
|
||||
recovery_conf:
|
||||
restore_command: cp ../wal_archive/%f %p
|
||||
parameters:
|
||||
unix_socket_directories: '.'
|
||||
archive_mode: "on"
|
||||
wal_level: logical
|
||||
archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
|
||||
max_wal_senders: 10
|
||||
wal_keep_segments: 8
|
||||
archive_timeout: 1800s
|
||||
max_replication_slots: 10
|
||||
max_worker_processes: 10
|
||||
track_commit_timestamp: 'on'
|
||||
shared_preload_libraries: 'bdr'
|
||||
hot_standby: "on"
|
||||
wal_log_hints: "on"
|
||||
tags:
|
||||
nofailover: false
|
||||
noloadbalance: false
|
||||
clonefrom: false
|
||||
nofailover: False
|
||||
noloadbalance: False
|
||||
clonefrom: False
|
||||
replicatefrom: 127.0.0.1
|
||||
|
||||
+19
-17
@@ -4,7 +4,7 @@ scope: &scope batman
|
||||
restapi:
|
||||
listen: 127.0.0.1:8009
|
||||
connect_address: 127.0.0.1:8009
|
||||
# auth: 'username:password'
|
||||
#auth: 'username:password'
|
||||
# certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
|
||||
# keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
|
||||
etcd:
|
||||
@@ -12,10 +12,6 @@ etcd:
|
||||
ttl: *ttl
|
||||
host: 127.0.0.1:4001
|
||||
#discovery_srv: my-etcd.domain
|
||||
#consul:
|
||||
# scope: *scope
|
||||
# ttl: *ttl
|
||||
# host: 127.0.0.1:8500
|
||||
#zookeeper:
|
||||
# scope: *scope
|
||||
# session_timeout: *ttl
|
||||
@@ -23,13 +19,16 @@ etcd:
|
||||
# hosts:
|
||||
# - 127.0.0.1:2181
|
||||
# - 127.0.0.2:2181
|
||||
#exhibitor:
|
||||
# poll_interval: 300
|
||||
# port: 8181
|
||||
# hosts:
|
||||
# - host1
|
||||
# - host2
|
||||
# - host3
|
||||
# exhibitor:
|
||||
# poll_interval: 300
|
||||
# port: 8181
|
||||
# hosts:
|
||||
# - host1
|
||||
# - host2
|
||||
# - host3
|
||||
bdr:
|
||||
enable: "on"
|
||||
database: "bdrtest"
|
||||
postgresql:
|
||||
name: postgresql1
|
||||
scope: *scope
|
||||
@@ -60,14 +59,14 @@ postgresql:
|
||||
username: postgres
|
||||
password: zalando
|
||||
pg_hba:
|
||||
- host replication replicator 127.0.0.1/32 md5
|
||||
- host all all 0.0.0.0/0 md5
|
||||
# - hostssl all all 0.0.0.0/0 md5
|
||||
- hostssl all all 0.0.0.0/0 md5
|
||||
replication:
|
||||
username: replicator
|
||||
password: rep-pass
|
||||
network: 127.0.0.1/32
|
||||
superuser:
|
||||
username: postgres
|
||||
user: postgres
|
||||
password: zalando
|
||||
admin:
|
||||
username: admin
|
||||
@@ -90,16 +89,19 @@ postgresql:
|
||||
restore_command: cp ../wal_archive/%f %p
|
||||
parameters:
|
||||
archive_mode: "on"
|
||||
wal_level: hot_standby
|
||||
wal_level: logical
|
||||
archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
|
||||
max_wal_senders: 10
|
||||
max_worker_processes: 10
|
||||
track_commit_timestamp: 'on'
|
||||
shared_preload_libraries: 'bdr'
|
||||
wal_keep_segments: 8
|
||||
archive_timeout: 1800s
|
||||
max_replication_slots: 10
|
||||
hot_standby: "on"
|
||||
wal_log_hints: "on"
|
||||
unix_socket_directories: '.'
|
||||
tags:
|
||||
nofailover: False
|
||||
noloadbalance: False
|
||||
clonefrom: False
|
||||
replicatefrom: 127.0.0.1
|
||||
|
||||
-106
@@ -1,106 +0,0 @@
|
||||
ttl: &ttl 30
|
||||
loop_wait: &loop_wait 10
|
||||
scope: &scope batman
|
||||
restapi:
|
||||
listen: 127.0.0.1:8010
|
||||
connect_address: 127.0.0.1:8010
|
||||
auth: 'username:password'
|
||||
# certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
|
||||
# keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
|
||||
etcd:
|
||||
scope: *scope
|
||||
ttl: *ttl
|
||||
host: 127.0.0.1:4001
|
||||
#discovery_srv: my-etcd.domain
|
||||
#consul:
|
||||
# scope: *scope
|
||||
# ttl: *ttl
|
||||
# host: 127.0.0.1:8500
|
||||
#zookeeper:
|
||||
# scope: *scope
|
||||
# session_timeout: *ttl
|
||||
# reconnect_timeout: *loop_wait
|
||||
# hosts:
|
||||
# - 127.0.0.1:2181
|
||||
# - 127.0.0.2:2181
|
||||
#exhibitor:
|
||||
# poll_interval: 300
|
||||
# port: 8181
|
||||
# hosts:
|
||||
# - host1
|
||||
# - host2
|
||||
# - host3
|
||||
postgresql:
|
||||
name: postgresql2
|
||||
scope: *scope
|
||||
listen: 127.0.0.1:5434
|
||||
connect_address: 127.0.0.1:5434
|
||||
data_dir: data/postgresql2
|
||||
maximum_lag_on_failover: 1048576 # 1 megabyte in bytes
|
||||
use_slots: True
|
||||
pgpass: /tmp/pgpass2
|
||||
initdb: ## We allow the following options to be passed on to initdb
|
||||
# - auth: authmethod
|
||||
# - auth-host: authmethod
|
||||
# - auth-local: authmethod
|
||||
- encoding: UTF8
|
||||
# - data-checksums # When pg_rewind is needed on 9.3, this needs to be enabled
|
||||
# - locale: locale
|
||||
# - lc-collate: locale
|
||||
# - lc-ctype: locale
|
||||
# - lc-messages: locale
|
||||
# - lc-monetary: locale
|
||||
# - lc-numeric: locale
|
||||
# - lc-time: locale
|
||||
# - text-search-config: CFG
|
||||
# - xlogdir: directory
|
||||
# - debug
|
||||
# - noclean
|
||||
pg_rewind:
|
||||
username: postgres
|
||||
password: zalando
|
||||
pg_hba:
|
||||
- host replication replicator 127.0.0.1/32 md5
|
||||
- host all all 0.0.0.0/0 md5
|
||||
# - hostssl all all 0.0.0.0/0 md5
|
||||
replication:
|
||||
username: replicator
|
||||
password: rep-pass
|
||||
superuser:
|
||||
username: postgres
|
||||
password: zalando
|
||||
admin:
|
||||
username: admin
|
||||
password: admin
|
||||
# commented-out example for wal-e provisioning
|
||||
create_replica_method:
|
||||
- basebackup
|
||||
# - wal_e
|
||||
# commented-out example for wal-e provisioning
|
||||
#wal_e:
|
||||
#command: /patroni/scripts/wale_restore.py
|
||||
#env_dir: /home/postgres/etc/wal-e.d/env
|
||||
#threshold_megabytes: 10240
|
||||
#threshold_backup_size_percentage: 30
|
||||
#retries: 2
|
||||
#use_iam: 1
|
||||
#recovery_conf:
|
||||
#restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1
|
||||
recovery_conf:
|
||||
restore_command: cp ../wal_archive/%f %p
|
||||
parameters:
|
||||
archive_mode: "on"
|
||||
wal_level: hot_standby
|
||||
archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
|
||||
max_wal_senders: 10
|
||||
wal_keep_segments: 8
|
||||
archive_timeout: 1800s
|
||||
max_replication_slots: 10
|
||||
hot_standby: "on"
|
||||
wal_log_hints: "on"
|
||||
unix_socket_directories: '.'
|
||||
tags:
|
||||
nofailover: False
|
||||
noloadbalance: False
|
||||
clonefrom: False
|
||||
replicatefrom: postgresql1
|
||||
@@ -0,0 +1,11 @@
|
||||
boto
|
||||
dnspython
|
||||
mock
|
||||
psycopg2>=2.6.1
|
||||
PyYAML
|
||||
requests
|
||||
six >= 1.7
|
||||
kazoo>=2.2.1
|
||||
python-etcd>=0.4.1
|
||||
click>=4.1
|
||||
prettytable>=0.7
|
||||
@@ -0,0 +1,11 @@
|
||||
boto
|
||||
mock
|
||||
dnspython3
|
||||
psycopg2>=2.6.1
|
||||
PyYAML
|
||||
requests
|
||||
six
|
||||
kazoo>=2.2.1
|
||||
python-etcd>=0.4.1
|
||||
click>=4.1
|
||||
prettytable>=0.7
|
||||
@@ -1,12 +0,0 @@
|
||||
boto
|
||||
psycopg2>=2.6.1
|
||||
PyYAML
|
||||
requests
|
||||
six >= 1.7
|
||||
kazoo==2.2.1
|
||||
python-etcd==0.4.3
|
||||
python-consul==0.6.0
|
||||
click>=4.1
|
||||
prettytable>=0.7
|
||||
tzlocal
|
||||
python-dateutil
|
||||
@@ -4,12 +4,13 @@
|
||||
Setup file for patroni
|
||||
"""
|
||||
|
||||
import inspect
|
||||
import os
|
||||
import sys
|
||||
import os
|
||||
import inspect
|
||||
|
||||
import setuptools
|
||||
from setuptools.command.test import test as TestCommand
|
||||
from setuptools import find_packages, setup
|
||||
from setuptools import setup
|
||||
|
||||
if sys.version_info < (2, 7, 0):
|
||||
sys.stderr.write('FATAL: patroni needs to be run with Python 2.7+\n')
|
||||
@@ -33,7 +34,7 @@ LICENSE = 'The MIT License'
|
||||
URL = 'https://github.com/zalando/patroni'
|
||||
AUTHOR = 'Alexander Kukushkin, Oleksii Kliukin, Feike Steenbergen'
|
||||
AUTHOR_EMAIL = '[email protected], [email protected], [email protected]'
|
||||
KEYWORDS = 'etcd governor patroni postgresql postgres ha haproxy confd zookeeper exhibitor consul streaming replication'
|
||||
KEYWORDS = 'etcd governor patroni postgresql postgres ha zookeeper streaming replication'
|
||||
|
||||
COVERAGE_XML = True
|
||||
COVERAGE_HTML = False
|
||||
@@ -50,8 +51,8 @@ CLASSIFIERS = [
|
||||
'Operating System :: POSIX :: Linux',
|
||||
'Programming Language :: Python',
|
||||
'Programming Language :: Python :: 2.7',
|
||||
'Programming Language :: Python :: 3.3',
|
||||
'Programming Language :: Python :: 3.4',
|
||||
'Programming Language :: Python :: 3.5',
|
||||
'Programming Language :: Python :: Implementation :: CPython',
|
||||
]
|
||||
|
||||
@@ -75,7 +76,7 @@ class PyTest(TestCommand):
|
||||
def finalize_options(self):
|
||||
TestCommand.finalize_options(self)
|
||||
if self.cov_xml or self.cov_html:
|
||||
self.cov = ['--cov', MAIN_PACKAGE, '--cov-report', 'term-missing']
|
||||
self.cov = ['--cov', MAIN_PACKAGE, '--cov', MAIN_PACKAGE, '--cov-report', 'term-missing']
|
||||
if self.cov_xml:
|
||||
self.cov.extend(['--cov-report', 'xml'])
|
||||
if self.cov_html:
|
||||
@@ -93,12 +94,7 @@ class PyTest(TestCommand):
|
||||
params['args'] += self.cov
|
||||
if self.junitxml:
|
||||
params['args'] += self.junitxml
|
||||
params['args'] += ['--doctest-modules', MAIN_PACKAGE, '-vv']
|
||||
|
||||
import logging
|
||||
silence = logging.WARNING
|
||||
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=os.getenv('LOGLEVEL', silence))
|
||||
params['args'] += ['-s' if logging.getLogger().getEffectiveLevel() < silence else '--capture=fd']
|
||||
params['args'] += ['--doctest-modules', MAIN_PACKAGE, '-s', '-vv']
|
||||
errno = pytest.main(**params)
|
||||
sys.exit(errno)
|
||||
|
||||
@@ -120,7 +116,8 @@ def setup_package():
|
||||
# Some helper variables
|
||||
version = os.getenv('GO_PIPELINE_LABEL', VERSION)
|
||||
|
||||
install_reqs = get_install_requirements('requirements.txt')
|
||||
requirements = 'requirements-py2.txt' if sys.version_info[0] == 2 else 'requirements-py3.txt'
|
||||
install_reqs = get_install_requirements(requirements)
|
||||
|
||||
command_options = {'test': {'test_suite': ('setup.py', 'tests')}}
|
||||
if JUNIT_XML:
|
||||
@@ -142,12 +139,12 @@ def setup_package():
|
||||
long_description=read('README.rst'),
|
||||
classifiers=CLASSIFIERS,
|
||||
test_suite='tests',
|
||||
packages=find_packages(exclude=['tests', 'tests.*']),
|
||||
packages=setuptools.find_packages(exclude=['tests', 'tests.*']),
|
||||
package_data={MAIN_PACKAGE: ["*.json"]},
|
||||
install_requires=install_reqs,
|
||||
setup_requires=['flake8'],
|
||||
setup_requires=['six', 'flake8'],
|
||||
cmdclass=cmdclass,
|
||||
tests_require=['mock>=2.0.0', 'pytest-cov', 'pytest'],
|
||||
tests_require=['pytest-cov', 'pytest'],
|
||||
command_options=command_options,
|
||||
entry_points={'console_scripts': CONSOLE_SCRIPTS},
|
||||
)
|
||||
|
||||
+59
-141
@@ -1,250 +1,168 @@
|
||||
import json
|
||||
import psycopg2
|
||||
import unittest
|
||||
|
||||
from mock import Mock, patch
|
||||
from patroni.api import RestApiHandler, RestApiServer
|
||||
from patroni.dcs import ClusterConfig, Member
|
||||
from patroni.dcs import Member
|
||||
from six import BytesIO as IO
|
||||
from six.moves import BaseHTTPServer
|
||||
from six.moves.BaseHTTPServer import BaseHTTPRequestHandler
|
||||
import socket
|
||||
from test_postgresql import psycopg2_connect, MockCursor
|
||||
|
||||
|
||||
class MockPostgresql(object):
|
||||
class MockPostgresql(Mock):
|
||||
|
||||
name = 'test'
|
||||
state = 'running'
|
||||
role = 'master'
|
||||
server_version = '999999'
|
||||
sysid = 'dummysysid'
|
||||
scope = 'dummy'
|
||||
pending_restart = True
|
||||
|
||||
@staticmethod
|
||||
def connection():
|
||||
def connection(self):
|
||||
return psycopg2_connect()
|
||||
|
||||
def is_running(self):
|
||||
return True
|
||||
|
||||
class MockHa(object):
|
||||
|
||||
class MockHa(Mock):
|
||||
|
||||
dcs = Mock()
|
||||
state_handler = MockPostgresql()
|
||||
|
||||
@staticmethod
|
||||
def schedule_reinitialize():
|
||||
def schedule_restart(self):
|
||||
return 'restart'
|
||||
|
||||
def schedule_reinitialize(self):
|
||||
return 'reinitialize'
|
||||
|
||||
@staticmethod
|
||||
def restart():
|
||||
def restart(self):
|
||||
return (True, '')
|
||||
|
||||
@staticmethod
|
||||
def restart_scheduled():
|
||||
def restart_scheduled(self):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def fetch_nodes_statuses(members):
|
||||
def fetch_nodes_statuses(self, members):
|
||||
return [[None, True, None, None, {}]]
|
||||
|
||||
|
||||
class MockPatroni(object):
|
||||
class MockPatroni:
|
||||
|
||||
nap_time = 10
|
||||
config = Mock()
|
||||
postgresql = MockPostgresql()
|
||||
ha = MockHa()
|
||||
dcs = Mock()
|
||||
tags = {}
|
||||
version = '0.00'
|
||||
noloadbalance = Mock(return_value=False)
|
||||
|
||||
@staticmethod
|
||||
def sighup_handler():
|
||||
pass
|
||||
|
||||
|
||||
class MockRequest(object):
|
||||
class MockRequest:
|
||||
|
||||
def __init__(self, request):
|
||||
self.request = request.encode('utf-8')
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
|
||||
def makefile(self, *args, **kwargs):
|
||||
return IO(self.request)
|
||||
return IO(self.path)
|
||||
|
||||
|
||||
class MockRestApiServer(RestApiServer):
|
||||
|
||||
def __init__(self, Handler, request):
|
||||
def __init__(self, Handler, path):
|
||||
self.socket = 0
|
||||
self.serve_forever = Mock()
|
||||
BaseHTTPServer.HTTPServer.__init__ = Mock()
|
||||
MockRestApiServer._BaseServer__is_shut_down = Mock()
|
||||
MockRestApiServer._BaseServer__shutdown_request = True
|
||||
config = {'listen': '127.0.0.1:8008', 'auth': 'test:test'}
|
||||
config = {'listen': '127.0.0.1:8008', 'auth': 'test:test', 'certfile': 'dumb'}
|
||||
super(MockRestApiServer, self).__init__(MockPatroni(), config)
|
||||
config['certfile'] = 'dumb'
|
||||
self.reload_config(config)
|
||||
Handler(MockRequest(request), ('0.0.0.0', 8080), self)
|
||||
Handler(MockRequest(path), ('0.0.0.0', 8080), self)
|
||||
|
||||
|
||||
@patch('ssl.wrap_socket', Mock(return_value=0))
|
||||
class TestRestApiHandler(unittest.TestCase):
|
||||
|
||||
_authorization = '\nAuthorization: Basic dGVzdDp0ZXN0'
|
||||
|
||||
def test_do_GET(self):
|
||||
MockRestApiServer(RestApiHandler, 'GET /replica')
|
||||
MockRestApiServer(RestApiHandler, b'GET /replica')
|
||||
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={})):
|
||||
MockRestApiServer(RestApiHandler, 'GET /replica')
|
||||
MockRestApiServer(RestApiHandler, b'GET /replica')
|
||||
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'master'})):
|
||||
MockRestApiServer(RestApiHandler, 'GET /replica')
|
||||
MockRestApiServer(RestApiHandler, 'GET /master')
|
||||
MockRestApiServer(RestApiHandler, b'GET /replica')
|
||||
MockRestApiServer(RestApiHandler, b'GET /master')
|
||||
MockPatroni.dcs.cluster.leader.name = MockPostgresql.name
|
||||
MockRestApiServer(RestApiHandler, 'GET /replica')
|
||||
MockRestApiServer(RestApiHandler, b'GET /replica')
|
||||
MockPatroni.dcs.cluster = None
|
||||
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'master'})):
|
||||
MockRestApiServer(RestApiHandler, 'GET /master')
|
||||
MockRestApiServer(RestApiHandler, b'GET /master')
|
||||
with patch.object(MockHa, 'restart_scheduled', Mock(return_value=True)):
|
||||
MockRestApiServer(RestApiHandler, 'GET /master')
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /master'))
|
||||
MockRestApiServer(RestApiHandler, b'GET /master')
|
||||
MockRestApiServer(RestApiHandler, b'GET /master')
|
||||
|
||||
def test_do_OPTIONS(self):
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'OPTIONS / HTTP/1.0'))
|
||||
MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0')
|
||||
|
||||
with patch.object(BaseHTTPRequestHandler, 'handle_one_request') as mock_handle_request:
|
||||
mock_handle_request.side_effect = socket.error("foo")
|
||||
MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0')
|
||||
|
||||
# make sure socket.error gets propagated via wfile object in finalize()
|
||||
with patch.object(MockRequest, 'makefile') as makefile:
|
||||
makefile.return_value.closed = False
|
||||
makefile.return_value.readline.side_effect = lambda x: b"foo"
|
||||
makefile.return_value.flush = Mock(side_effect=socket.error("foo"))
|
||||
MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0')
|
||||
|
||||
def test_do_GET_patroni(self):
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
|
||||
MockRestApiServer(RestApiHandler, b'GET /patroni')
|
||||
|
||||
def test_basicauth(self):
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0'))
|
||||
MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0\nAuthorization:')
|
||||
|
||||
@patch.object(MockHa, 'dcs')
|
||||
def test_do_GET_config(self, mock_dcs):
|
||||
mock_dcs.cluster.config.data = {}
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /config'))
|
||||
mock_dcs.cluster.config = None
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /config'))
|
||||
|
||||
@patch.object(MockHa, 'dcs')
|
||||
def test_do_PATCH_config(self, mock_dcs):
|
||||
config = {'postgresql': {'use_slots': False, 'use_pg_rewind': True, 'parameters': {'wal_level': 'logical'}}}
|
||||
mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, json.dumps(config))
|
||||
request = 'PATCH /config HTTP/1.0' + self._authorization
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
|
||||
request += '\nContent-Length: '
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '34\n\n{"postgresql":{"use_slots":false}}'))
|
||||
config['ttl'] = 5
|
||||
config['postgresql'].update({'use_slots': {'foo': True}, "parameters": None})
|
||||
config = json.dumps(config)
|
||||
request += str(len(config)) + '\n\n' + config
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
mock_dcs.set_config_value.return_value = False
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
@patch.object(MockHa, 'dcs')
|
||||
def test_do_PUT_config(self, mock_dcs):
|
||||
mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, '{}')
|
||||
request = 'PUT /config HTTP/1.0' + self._authorization + '\nContent-Length: '
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '2\n\n{}'))
|
||||
config = '{"foo": "bar"}'
|
||||
request += str(len(config)) + '\n\n' + config
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
mock_dcs.set_config_value.return_value = False
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, config)
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
@patch.object(MockPatroni, 'sighup_handler', Mock(side_effect=Exception))
|
||||
def test_do_POST_reload(self):
|
||||
with patch.object(MockPatroni, 'config') as mock_config:
|
||||
mock_config.reload_local_configuration.return_value = False
|
||||
MockRestApiServer(RestApiHandler, 'POST /reload HTTP/1.0' + self._authorization)
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /reload HTTP/1.0' + self._authorization))
|
||||
MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0')
|
||||
MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0\nAuthorization:')
|
||||
|
||||
def test_do_POST_restart(self):
|
||||
request = 'POST /restart HTTP/1.0' + self._authorization
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
|
||||
request = b'POST /restart HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
with patch.object(MockHa, 'restart', Mock(side_effect=Exception)):
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
@patch.object(MockHa, 'dcs')
|
||||
def test_do_POST_reinitialize(self, dcs):
|
||||
cluster = dcs.get_cluster.return_value
|
||||
request = 'POST /reinitialize HTTP/1.0' + self._authorization
|
||||
request = b'POST /reinitialize HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
cluster.is_unlocked.return_value = False
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
with patch.object(MockHa, 'schedule_reinitialize', Mock(return_value=None)):
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
cluster.leader.name = 'test'
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
@patch('time.sleep', Mock())
|
||||
def test_RestApiServer_query(self):
|
||||
with patch.object(MockCursor, 'execute', Mock(side_effect=psycopg2.OperationalError)):
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
|
||||
MockRestApiServer(RestApiHandler, b'GET /patroni')
|
||||
with patch.object(MockPostgresql, 'connection', Mock(side_effect=psycopg2.OperationalError)):
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
|
||||
MockRestApiServer(RestApiHandler, b'GET /patroni')
|
||||
|
||||
@patch('time.sleep', Mock())
|
||||
@patch.object(MockHa, 'dcs')
|
||||
def test_do_POST_failover(self, dcs):
|
||||
cluster = dcs.get_cluster.return_value
|
||||
|
||||
post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
|
||||
|
||||
MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}')
|
||||
|
||||
request = post + '0\n\n'
|
||||
request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
|
||||
b'Content-Length: 25\n\n{"leader": "postgresql1"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
cluster.leader.name = 'postgresql1'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
MockRestApiServer(RestApiHandler, post + '25\n\n{"leader": "postgresql1"}')
|
||||
|
||||
cluster.leader.name = 'postgresql2'
|
||||
request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
cluster.leader.name = 'postgresql1'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
|
||||
Member(0, 'postgresql2', 30, {'api_url': 'http'})]
|
||||
cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'})]
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
with patch.object(MockPatroni, 'dcs') as d:
|
||||
cluster = d.get_cluster.return_value
|
||||
cluster.leader.name = 'postgresql0'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
cluster.leader.name = 'postgresql2'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
cluster.leader.name = 'postgresql1'
|
||||
cluster.failover = None
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
d.get_cluster = Mock(side_effect=Exception)
|
||||
d.get_cluster = Mock(side_effect=Exception())
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
d.manual_failover.return_value = False
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
with patch.object(MockHa, 'fetch_nodes_statuses', Mock(return_value=[])):
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
# Valid future date
|
||||
request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2",' +\
|
||||
' "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}'
|
||||
request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
|
||||
b'Content-Length: 50\n\n{"leader": "postgresql1", "member": "postgresql2"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
with patch.object(MockPatroni, 'dcs') as d:
|
||||
d.manual_failover.return_value = False
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
# Exception: No timezone specified
|
||||
request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' +\
|
||||
' "scheduled_at": "6016-02-15T18:13:30.568224"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
# Exception: Scheduled in the past
|
||||
request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "'
|
||||
MockRestApiServer(RestApiHandler, request + '1016-02-15T18:13:30.568224+01:00"}')
|
||||
|
||||
# Invalid date
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}'))
|
||||
|
||||
+18
-18
@@ -1,15 +1,12 @@
|
||||
import boto.ec2
|
||||
import requests
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
from mock import Mock, patch
|
||||
import requests
|
||||
import boto.ec2
|
||||
from collections import namedtuple
|
||||
from patroni.scripts.aws import AWSConnection, main as _main
|
||||
from patroni.scripts.aws import AWSConnection
|
||||
from requests.exceptions import RequestException
|
||||
|
||||
|
||||
class MockEc2Connection(object):
|
||||
class MockEc2Connection:
|
||||
|
||||
def __init__(self, error=False):
|
||||
self.error = error
|
||||
@@ -26,7 +23,7 @@ class MockEc2Connection(object):
|
||||
return True
|
||||
|
||||
|
||||
class MockResponse(object):
|
||||
class MockResponse:
|
||||
|
||||
def __init__(self, content):
|
||||
self.content = content
|
||||
@@ -38,6 +35,15 @@ class MockResponse(object):
|
||||
|
||||
class TestAWSConnection(unittest.TestCase):
|
||||
|
||||
def __init__(self, method_name='runTest'):
|
||||
super(TestAWSConnection, self).__init__(method_name)
|
||||
|
||||
def set_error(self):
|
||||
self.error = True
|
||||
|
||||
def set_json_error(self):
|
||||
self.json_error = True
|
||||
|
||||
def boto_ec2_connect_to_region(self, region):
|
||||
return MockEc2Connection(self.error)
|
||||
|
||||
@@ -68,27 +74,21 @@ class TestAWSConnection(unittest.TestCase):
|
||||
self.assertTrue(self.conn.on_role_change('master'))
|
||||
|
||||
def test_non_aws(self):
|
||||
self.error = True
|
||||
self.set_error()
|
||||
conn = AWSConnection('test')
|
||||
self.assertFalse(conn.aws_available())
|
||||
self.assertFalse(conn._tag_ebs('master'))
|
||||
self.assertFalse(conn._tag_ec2('master'))
|
||||
|
||||
def test_aws_bizare_response(self):
|
||||
self.json_error = True
|
||||
self.set_json_error()
|
||||
conn = AWSConnection('test')
|
||||
self.assertFalse(conn.aws_available())
|
||||
|
||||
def test_aws_tag_ebs_error(self):
|
||||
self.error = True
|
||||
self.set_error()
|
||||
self.assertFalse(self.conn._tag_ebs("master"))
|
||||
|
||||
def test_aws_tag_ec2_error(self):
|
||||
self.error = True
|
||||
self.set_error()
|
||||
self.assertFalse(self.conn._tag_ec2("master"))
|
||||
|
||||
@patch('sys.exit', Mock())
|
||||
def test_main(self):
|
||||
self.assertIsNone(_main())
|
||||
sys.argv = ['aws.py', 'on_start', 'replica', 'foo']
|
||||
self.assertIsNone(_main())
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
import os
|
||||
import unittest
|
||||
import sys
|
||||
|
||||
from mock import MagicMock, Mock, patch
|
||||
from patroni.config import Config
|
||||
from six.moves import builtins
|
||||
|
||||
|
||||
class TestConfig(unittest.TestCase):
|
||||
|
||||
@patch('os.path.isfile', Mock(return_value=True))
|
||||
@patch('json.load', Mock(side_effect=Exception))
|
||||
@patch.object(builtins, 'open', MagicMock())
|
||||
def setUp(self):
|
||||
sys.argv = ['patroni.py']
|
||||
os.environ[Config.PATRONI_CONFIG_VARIABLE] = 'restapi: {}\npostgresql: {data_dir: foo}'
|
||||
self.config = Config()
|
||||
|
||||
def test_no_config(self):
|
||||
self.assertRaises(SystemExit, Config)
|
||||
|
||||
@patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception))
|
||||
def test_set_dynamic_configuration(self):
|
||||
self.assertIsNone(self.config.set_dynamic_configuration({'foo': 'bar'}))
|
||||
|
||||
def test_reload_local_configuration(self):
|
||||
os.environ.update({
|
||||
'PATRONI_NAME': 'postgres0',
|
||||
'PATRONI_NAMESPACE': '/patroni/',
|
||||
'PATRONI_SCOPE': 'batman2',
|
||||
'PATRONI_RESTAPI_USERNAME': 'username',
|
||||
'PATRONI_RESTAPI_PASSWORD': 'password',
|
||||
'PATRONI_RESTAPI_LISTEN': '0.0.0.0:8008',
|
||||
'PATRONI_RESTAPI_CONNECT_ADDRESS': '127.0.0.1:8008',
|
||||
'PATRONI_RESTAPI_CERTFILE': '/certfile',
|
||||
'PATRONI_RESTAPI_KEYFILE': '/keyfile',
|
||||
'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432',
|
||||
'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432',
|
||||
'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
|
||||
'PATRONI_POSTGRESQL_PGPASS': '/tmp/pgpass0',
|
||||
'PATRONI_ETCD_HOST': '127.0.0.1:2379',
|
||||
'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
|
||||
'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
|
||||
'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',
|
||||
'PATRONI_EXHIBITOR_PORT': '8181',
|
||||
'PATRONI_foo_HOSTS': '[host1,host2', # Exception in parse_list
|
||||
'PATRONI_SUPERUSER_USERNAME': 'postgres',
|
||||
'PATRONI_SUPERUSER_PASSWORD': 'zalando',
|
||||
'PATRONI_REPLICATION_USERNAME': 'replicator',
|
||||
'PATRONI_REPLICATION_PASSWORD': 'rep-pass',
|
||||
'PATRONI_admin_PASSWORD': 'admin',
|
||||
'PATRONI_admin_OPTIONS': 'createrole,createdb'
|
||||
})
|
||||
sys.argv = ['patroni.py', 'postgres0.yml']
|
||||
config = Config()
|
||||
with patch.object(Config, '_load_config_file', Mock(return_value={'restapi': {}})):
|
||||
with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
|
||||
self.assertRaises(Exception, config.reload_local_configuration, True)
|
||||
self.assertTrue(config.reload_local_configuration(True))
|
||||
self.assertTrue(config.reload_local_configuration())
|
||||
|
||||
@patch('tempfile.mkstemp', Mock(return_value=[3000, 'blabla']))
|
||||
@patch('os.path.exists', Mock(return_value=True))
|
||||
@patch('os.remove', Mock(side_effect=IOError))
|
||||
@patch('os.close', Mock(side_effect=IOError))
|
||||
@patch('os.rename', Mock(return_value=None))
|
||||
@patch('json.dump', Mock())
|
||||
def test_save_cache(self):
|
||||
self.config.set_dynamic_configuration({'ttl': 30, 'postgresql': {'foo': 'bar'}})
|
||||
with patch('os.fdopen', Mock(side_effect=IOError)):
|
||||
self.config.save_cache()
|
||||
with patch('os.fdopen', MagicMock()):
|
||||
self.config.save_cache()
|
||||
@@ -1,140 +0,0 @@
|
||||
import consul
|
||||
import unittest
|
||||
|
||||
from mock import Mock, patch
|
||||
from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulError, ConsulException, HTTPClient, NotFound
|
||||
from test_etcd import SleepException
|
||||
|
||||
|
||||
def kv_get(self, key, **kwargs):
|
||||
if key == 'service/test/members/postgresql1':
|
||||
return '1', {'Session': 'fd4f44fe-2cac-bba5-a60b-304b51ff39b7'}
|
||||
if key == 'service/test/':
|
||||
return None, None
|
||||
if key == 'service/good/leader':
|
||||
return '1', None
|
||||
if key == 'service/good/':
|
||||
return ('6429',
|
||||
[{'CreateIndex': 1334, 'Flags': 0, 'Key': key + 'failover', 'LockIndex': 0,
|
||||
'ModifyIndex': 1334, 'Value': b''},
|
||||
{'CreateIndex': 1334, 'Flags': 0, 'Key': key + 'initialize', 'LockIndex': 0,
|
||||
'ModifyIndex': 1334, 'Value': b'postgresql0'},
|
||||
{'CreateIndex': 2621, 'Flags': 0, 'Key': key + 'leader', 'LockIndex': 1,
|
||||
'ModifyIndex': 2621, 'Session': 'fd4f44fe-2cac-bba5-a60b-304b51ff39b7', 'Value': b'postgresql1'},
|
||||
{'CreateIndex': 6156, 'Flags': 0, 'Key': key + 'members/postgresql0', 'LockIndex': 1,
|
||||
'ModifyIndex': 6156, 'Session': '782e6da4-ed02-3aef-7963-99a90ed94b53',
|
||||
'Value': ('postgres://replicator:[email protected]:5432/postgres' +
|
||||
'?application_name=http://127.0.0.1:8008/patroni').encode('utf-8')},
|
||||
{'CreateIndex': 2630, 'Flags': 0, 'Key': key + 'members/postgresql1', 'LockIndex': 1,
|
||||
'ModifyIndex': 2630, 'Session': 'fd4f44fe-2cac-bba5-a60b-304b51ff39b7',
|
||||
'Value': ('postgres://replicator:[email protected]:5433/postgres' +
|
||||
'?application_name=http://127.0.0.1:8009/patroni').encode('utf-8')},
|
||||
{'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'optime/leader', 'LockIndex': 0,
|
||||
'ModifyIndex': 6429, 'Value': b'4496294792'}])
|
||||
raise ConsulException
|
||||
|
||||
|
||||
class TestHTTPClient(unittest.TestCase):
|
||||
|
||||
def test_get(self):
|
||||
self.client = HTTPClient('127.0.0.1', '8500', 'http', False)
|
||||
self.client.session.get = Mock()
|
||||
self.client.get(Mock(), '')
|
||||
self.client.get(Mock(), '', {'wait': '1s', 'index': 1})
|
||||
|
||||
|
||||
@patch.object(consul.Consul.KV, 'get', kv_get)
|
||||
class TestConsul(unittest.TestCase):
|
||||
|
||||
@patch.object(consul.Consul.Session, 'create', Mock(return_value='fd4f44fe-2cac-bba5-a60b-304b51ff39b7'))
|
||||
@patch.object(consul.Consul.Session, 'renew', Mock(side_effect=NotFound))
|
||||
@patch.object(consul.Consul.KV, 'get', kv_get)
|
||||
@patch.object(consul.Consul.KV, 'delete', Mock())
|
||||
def setUp(self):
|
||||
self.c = Consul({'ttl': 30, 'scope': 'test', 'name': 'postgresql1', 'host': 'localhost:1', 'retry_timeout': 10})
|
||||
self.c._base_path = '/service/good'
|
||||
self.c._load_cluster()
|
||||
|
||||
@patch('time.sleep', Mock(side_effect=SleepException))
|
||||
@patch.object(consul.Consul.Session, 'create', Mock(side_effect=ConsulException))
|
||||
def test_create_session(self):
|
||||
self.c._session = None
|
||||
self.assertRaises(SleepException, self.c.create_session)
|
||||
|
||||
@patch.object(consul.Consul.Session, 'renew', Mock(side_effect=NotFound))
|
||||
@patch.object(consul.Consul.Session, 'create', Mock(side_effect=ConsulException))
|
||||
def test_referesh_session(self):
|
||||
self.c._session = '1'
|
||||
self.c._name = ''
|
||||
self.assertRaises(ConsulError, self.c.refresh_session)
|
||||
|
||||
@patch.object(consul.Consul.KV, 'delete', Mock())
|
||||
def test_get_cluster(self):
|
||||
self.c._base_path = '/service/test'
|
||||
self.assertIsInstance(self.c.get_cluster(), Cluster)
|
||||
self.assertIsInstance(self.c.get_cluster(), Cluster)
|
||||
self.c._base_path = '/service/fail'
|
||||
self.assertRaises(ConsulError, self.c.get_cluster)
|
||||
self.c._base_path = '/service/good'
|
||||
self.c._session = 'fd4f44fe-2cac-bba5-a60b-304b51ff39b8'
|
||||
self.assertIsInstance(self.c.get_cluster(), Cluster)
|
||||
|
||||
@patch.object(consul.Consul.KV, 'delete', Mock(side_effect=[ConsulException, True, True]))
|
||||
@patch.object(consul.Consul.KV, 'put', Mock(side_effect=[True, ConsulException]))
|
||||
def test_touch_member(self):
|
||||
self.c.refresh_session = Mock(return_value=True)
|
||||
self.c.touch_member('balbla')
|
||||
self.c.touch_member('balbla')
|
||||
self.c.touch_member('balbla')
|
||||
self.c.refresh_session = Mock(return_value=False)
|
||||
self.c.touch_member('balbla')
|
||||
|
||||
@patch.object(consul.Consul.KV, 'put', Mock(return_value=False))
|
||||
def test_take_leader(self):
|
||||
self.c.take_leader()
|
||||
|
||||
@patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
|
||||
def test_set_failover_value(self):
|
||||
self.c.set_failover_value('')
|
||||
|
||||
@patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
|
||||
def test_set_config_value(self):
|
||||
self.c.set_config_value('')
|
||||
|
||||
@patch.object(consul.Consul.KV, 'put', Mock(side_effect=ConsulException))
|
||||
def test_write_leader_optime(self):
|
||||
self.c.write_leader_optime('')
|
||||
|
||||
def test_update_leader(self):
|
||||
self.c.update_leader()
|
||||
|
||||
@patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
|
||||
def test_delete_leader(self):
|
||||
self.c.delete_leader()
|
||||
|
||||
@patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
|
||||
def test_initialize(self):
|
||||
self.c.initialize()
|
||||
|
||||
@patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
|
||||
def test_cancel_initialization(self):
|
||||
self.c.cancel_initialization()
|
||||
|
||||
@patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
|
||||
def test_delete_cluster(self):
|
||||
self.c.delete_cluster()
|
||||
|
||||
@patch.object(AbstractDCS, 'watch', Mock())
|
||||
def test_watch(self):
|
||||
self.c._name = ''
|
||||
self.c.watch(1)
|
||||
with patch.object(consul.Consul.KV, 'get', Mock(side_effect=ConsulException)):
|
||||
self.c.watch(1)
|
||||
|
||||
@patch.object(consul.Consul.Session, 'destroy', Mock(side_effect=ConsulException))
|
||||
def test_set_ttl(self):
|
||||
self.c.set_ttl(20)
|
||||
self.assertTrue(self.c.watch(1))
|
||||
|
||||
def test_set_retry_timeout(self):
|
||||
self.c.set_retry_timeout(10)
|
||||
+274
-192
@@ -1,294 +1,376 @@
|
||||
import etcd
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
import requests
|
||||
import sys
|
||||
import pytest
|
||||
import unittest
|
||||
import psycopg2
|
||||
import requests
|
||||
import patroni.exceptions
|
||||
import etcd
|
||||
from mock import patch, Mock
|
||||
|
||||
from click.testing import CliRunner
|
||||
from mock import patch, Mock
|
||||
from patroni.ctl import ctl, members, store_config, load_config, output_members, post_patroni, get_dcs, parse_dcs, \
|
||||
wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException
|
||||
from psycopg2 import OperationalError
|
||||
from test_etcd import etcd_read, requests_get, socket_getaddrinfo, MockResponse
|
||||
from patroni.ctl import ctl, members, store_config, load_config, output_members, post_patroni, get_dcs, \
|
||||
wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure
|
||||
from patroni.ha import Ha
|
||||
from patroni.etcd import Etcd, Client
|
||||
from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \
|
||||
get_cluster_initialized_with_only_leader
|
||||
get_cluster_initialized_with_only_leader, MockPostgresql, MockPatroni, run_async, \
|
||||
get_cluster_not_initialized_without_leader
|
||||
from test_etcd import etcd_read, etcd_write, requests_get, MockResponse
|
||||
from test_postgresql import MockConnect, psycopg2_connect
|
||||
|
||||
CONFIG_FILE_PATH = './test-ctl.yaml'
|
||||
|
||||
|
||||
def test_rw_config():
|
||||
runner = CliRunner()
|
||||
config = {'a':'b'}
|
||||
with runner.isolated_filesystem():
|
||||
store_config({'etcd': {'host': 'localhost:2379'}}, CONFIG_FILE_PATH + '/dummy')
|
||||
sys.argv = ['patronictl.py', '']
|
||||
load_config(CONFIG_FILE_PATH + '/dummy', None)
|
||||
load_config(CONFIG_FILE_PATH + '/dummy', '0.0.0.0')
|
||||
store_config(config, CONFIG_FILE_PATH + '/dummy')
|
||||
os.remove(CONFIG_FILE_PATH + '/dummy')
|
||||
os.rmdir(CONFIG_FILE_PATH)
|
||||
|
||||
with pytest.raises(Exception):
|
||||
result = load_config(CONFIG_FILE_PATH, None)
|
||||
assert 'Could not load configuration file' in result.output
|
||||
|
||||
@patch('patroni.ctl.load_config', Mock(return_value={'restapi': {'auth': 'u:p'}, 'etcd': {'host': 'localhost:4001'}}))
|
||||
os.mkdir(CONFIG_FILE_PATH)
|
||||
with pytest.raises(Exception):
|
||||
store_config(config, CONFIG_FILE_PATH)
|
||||
|
||||
os.rmdir(CONFIG_FILE_PATH)
|
||||
|
||||
store_config(config, CONFIG_FILE_PATH)
|
||||
load_config(CONFIG_FILE_PATH, None)
|
||||
load_config(CONFIG_FILE_PATH, '0.0.0.0')
|
||||
|
||||
@patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'etcd', 'hostname': 'localhost', 'port': 4001}}))
|
||||
class TestCtl(unittest.TestCase):
|
||||
|
||||
@patch('socket.getaddrinfo', socket_getaddrinfo)
|
||||
def setUp(self):
|
||||
self.runner = CliRunner()
|
||||
with patch.object(etcd.Client, 'machines') as mock_machines:
|
||||
mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
|
||||
self.e = get_dcs({'etcd': {'ttl': 30, 'host': 'ok:2379', 'retry_timeout': 10}}, 'foo')
|
||||
@patch.object(Client, 'machines')
|
||||
def setUp(self, mock_machines):
|
||||
mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
|
||||
self.p = MockPostgresql()
|
||||
self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
|
||||
self.e.client.read = etcd_read
|
||||
self.e.client.write = etcd_write
|
||||
self.e.client.delete = Mock(side_effect=etcd.EtcdException())
|
||||
self.ha = Ha(MockPatroni(self.p, self.e))
|
||||
self.ha._async_executor.run_async = run_async
|
||||
self.ha.old_cluster = self.e.get_cluster()
|
||||
self.ha.cluster = get_cluster_not_initialized_without_leader()
|
||||
self.ha.load_cluster_from_dcs = Mock()
|
||||
|
||||
@patch('psycopg2.connect', psycopg2_connect)
|
||||
def test_get_cursor(self):
|
||||
self.assertIsNone(get_cursor(get_cluster_initialized_without_leader(), {}, role='master'))
|
||||
c = get_cursor(get_cluster_initialized_without_leader(), role='master')
|
||||
assert c is None
|
||||
|
||||
self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), {}, role='master'))
|
||||
c = get_cursor(get_cluster_initialized_with_leader(), role='master')
|
||||
assert c is not None
|
||||
|
||||
# MockCursor returns pg_is_in_recovery as false
|
||||
self.assertIsNone(get_cursor(get_cluster_initialized_with_leader(), {}, role='replica'))
|
||||
c = get_cursor(get_cluster_initialized_with_leader(), role='replica')
|
||||
# # MockCursor returns pg_is_in_recovery as false
|
||||
assert c is None
|
||||
|
||||
self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), {'database': 'foo'}, role='any'))
|
||||
|
||||
def test_parse_dcs(self):
|
||||
assert parse_dcs(None) is None
|
||||
assert parse_dcs('localhost') == {'etcd': {'host': 'localhost:4001'}}
|
||||
assert parse_dcs('') == {'etcd': {'host': 'localhost:4001'}}
|
||||
assert parse_dcs('localhost:8500') == {'consul': {'host': 'localhost:8500'}}
|
||||
assert parse_dcs('zookeeper://localhost') == {'zookeeper': {'hosts': ['localhost:2181']}}
|
||||
assert parse_dcs('exhibitor://dummy') == {'exhibitor': {'hosts': ['dummy'], 'port': 8181}}
|
||||
assert parse_dcs('consul://localhost') == {'consul': {'host': 'localhost:8500'}}
|
||||
self.assertRaises(PatroniCtlException, parse_dcs, 'invalid://test')
|
||||
c = get_cursor(get_cluster_initialized_with_leader(), role='any')
|
||||
assert c is not None
|
||||
|
||||
def test_output_members(self):
|
||||
cluster = get_cluster_initialized_with_leader()
|
||||
self.assertIsNone(output_members(cluster, name='abc', fmt='pretty'))
|
||||
self.assertIsNone(output_members(cluster, name='abc', fmt='json'))
|
||||
self.assertIsNone(output_members(cluster, name='abc', fmt='tsv'))
|
||||
output_members(cluster, name='abc', format='pretty')
|
||||
output_members(cluster, name='abc', format='json')
|
||||
output_members(cluster, name='abc', format='tsv')
|
||||
|
||||
@patch('patroni.ctl.get_dcs')
|
||||
@patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
|
||||
@patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
|
||||
@patch('patroni.etcd.Etcd.set_failover_value', Mock(return_value=None))
|
||||
@patch('patroni.ctl.wait_for_leader', Mock(return_value=get_cluster_initialized_with_leader()))
|
||||
@patch('requests.get', requests_get)
|
||||
@patch('requests.post', requests_get)
|
||||
@patch('patroni.ctl.post_patroni', Mock(return_value=MockResponse()))
|
||||
def test_failover(self, mock_get_dcs):
|
||||
mock_get_dcs.return_value = self.e
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
|
||||
assert 'leader' in result.output
|
||||
def test_failover(self):
|
||||
runner = CliRunner()
|
||||
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2100-01-01T12:23:00\ny')
|
||||
assert result.exit_code == 0
|
||||
with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())):
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
other
|
||||
y''')
|
||||
assert 'Failing over to new leader' in result.output
|
||||
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2030-01-01T12:23:00\ny')
|
||||
assert result.exit_code == 0
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
other
|
||||
N''')
|
||||
assert 'Aborting failover' in str(result.exception)
|
||||
|
||||
# Aborting failover,as we anser NO to the confirmation
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\nN')
|
||||
assert result.exit_code == 1
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
leader
|
||||
y''')
|
||||
assert 'target and source are the same' in str(result.exception)
|
||||
|
||||
# Target and source are equal
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nleader\n\ny')
|
||||
assert result.exit_code == 1
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
Reality
|
||||
y''')
|
||||
assert 'Reality does not exist' in str(result.exception)
|
||||
|
||||
# Reality is not part of this cluster
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nReality\n\ny')
|
||||
assert result.exit_code == 1
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--force'])
|
||||
assert 'Failing over to new leader' in result.output
|
||||
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--force'])
|
||||
assert 'Member' in result.output
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='dummy')
|
||||
assert 'is not the leader of cluster' in str(result.exception)
|
||||
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
|
||||
assert result.exit_code == 0
|
||||
with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_only_leader())):
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
other
|
||||
y''')
|
||||
assert 'No candidates found to failover to' in str(result.exception)
|
||||
|
||||
# Invalid timestamp
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', 'invalid'])
|
||||
assert result.exit_code != 0
|
||||
with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_without_leader())):
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
other
|
||||
y''')
|
||||
assert 'This cluster has no master' in str(result.exception)
|
||||
|
||||
# Invalid timestamp
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2115-02-30T12:00:00+01:00'])
|
||||
assert result.exit_code != 0
|
||||
|
||||
# Specifying wrong leader
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='dummy')
|
||||
assert result.exit_code == 1
|
||||
|
||||
with patch('patroni.ctl.post_patroni', Mock(side_effect=Exception)):
|
||||
# Non-responding patroni
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
|
||||
with patch('patroni.ctl.post_patroni', Mock(side_effect=Exception())):
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
other
|
||||
y''')
|
||||
assert 'falling back to DCS' in result.output
|
||||
|
||||
with patch('patroni.ctl.post_patroni') as mocked:
|
||||
mocked.return_value.status_code = 500
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
|
||||
assert 'Failover failed' in result.output
|
||||
|
||||
# No members available
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
|
||||
assert result.exit_code == 1
|
||||
mocked = Mock()
|
||||
mocked.return_value.status_code = 500
|
||||
with patch('patroni.ctl.post_patroni', Mock(return_value=mocked)):
|
||||
result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
|
||||
other
|
||||
y''')
|
||||
assert 'Failover failed, details' in result.output
|
||||
|
||||
# No master available
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
|
||||
assert result.exit_code == 1
|
||||
# with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())):
|
||||
# result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='nonsense')
|
||||
# assert 'is not the leader of cluster' in str(result.exception)
|
||||
|
||||
def test_get_dcs(self):
|
||||
self.assertRaises(PatroniCtlException, get_dcs, {'dummy': {}}, 'dummy')
|
||||
# result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8', '--master', 'nonsense'])
|
||||
# assert 'is not the leader of cluster' in str(result.exception)
|
||||
|
||||
# result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='leader\nother\nn')
|
||||
# assert 'Aborting failover' in str(result.exception)
|
||||
|
||||
# with patch('patroni.ctl.wait_for_leader', Mock(return_value = get_cluster_initialized_with_leader())):
|
||||
# result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='leader\nother\nY')
|
||||
# assert 'master did not change after' in result.output
|
||||
|
||||
# result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='leader\nother\nY')
|
||||
# assert 'Failover failed' in result.output
|
||||
|
||||
def test_(self):
|
||||
self.assertRaises(patroni.exceptions.PatroniCtlException, get_dcs, {'scheme': 'dummy'}, 'dummy')
|
||||
|
||||
@patch('psycopg2.connect', psycopg2_connect)
|
||||
@patch('patroni.ctl.query_member', Mock(return_value=([['mock column']], None)))
|
||||
@patch('patroni.ctl.get_dcs')
|
||||
@patch.object(etcd.Client, 'read', etcd_read)
|
||||
def test_query(self, mock_get_dcs):
|
||||
mock_get_dcs.return_value = self.e
|
||||
# Mutually exclusive
|
||||
result = self.runner.invoke(ctl, ['query', 'alpha', '--member', 'abc', '--role', 'master'])
|
||||
assert result.exit_code == 1
|
||||
def test_query(self):
|
||||
runner = CliRunner()
|
||||
|
||||
with self.runner.isolated_filesystem():
|
||||
with open('dummy', 'w') as dummy_file:
|
||||
with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)):
|
||||
result = runner.invoke(ctl, [
|
||||
'query',
|
||||
'alpha',
|
||||
'--member',
|
||||
'abc',
|
||||
'--role',
|
||||
'master',
|
||||
])
|
||||
assert 'mutually exclusive' in str(result.exception)
|
||||
|
||||
with runner.isolated_filesystem():
|
||||
dummy_file = open('dummy', 'w')
|
||||
dummy_file.write('SELECT 1')
|
||||
dummy_file.close()
|
||||
|
||||
# Mutually exclusive
|
||||
result = self.runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy', '--command', 'dummy'])
|
||||
assert result.exit_code == 1
|
||||
result = runner.invoke(ctl, [
|
||||
'query',
|
||||
'alpha',
|
||||
'--file',
|
||||
'dummy',
|
||||
'--command',
|
||||
'dummy',
|
||||
])
|
||||
assert 'mutually exclusive' in str(result.exception)
|
||||
|
||||
result = self.runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy'])
|
||||
assert result.exit_code == 0
|
||||
result = runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy'])
|
||||
|
||||
os.remove('dummy')
|
||||
os.remove('dummy')
|
||||
|
||||
result = self.runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1'])
|
||||
assert 'mock column' in result.output
|
||||
|
||||
# --command or --file is mandatory
|
||||
result = self.runner.invoke(ctl, ['query', 'alpha'])
|
||||
assert result.exit_code == 1
|
||||
|
||||
result = self.runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1', '--username', 'root',
|
||||
'--password', '--dbname', 'postgres'], input='ab\nab')
|
||||
assert 'mock column' in result.output
|
||||
result = runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1'])
|
||||
assert 'mock column' in result.output
|
||||
|
||||
@patch('patroni.ctl.get_cursor', Mock(return_value=MockConnect().cursor()))
|
||||
def test_query_member(self):
|
||||
with patch('patroni.ctl.get_cursor', Mock(return_value=MockConnect().cursor())):
|
||||
rows = query_member(None, None, None, 'master', 'SELECT pg_is_in_recovery()', {})
|
||||
self.assertTrue('False' in str(rows))
|
||||
rows = query_member(None, None, None, 'master', 'SELECT pg_is_in_recovery()')
|
||||
assert 'False' in str(rows)
|
||||
|
||||
rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()', {})
|
||||
self.assertEquals(rows, (None, None))
|
||||
|
||||
with patch('test_postgresql.MockCursor.execute', Mock(side_effect=OperationalError('bla'))):
|
||||
rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()', {})
|
||||
rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
|
||||
assert rows == (None, None)
|
||||
|
||||
with patch('patroni.ctl.get_cursor', Mock(return_value=None)):
|
||||
rows = query_member(None, None, None, None, 'SELECT pg_is_in_recovery()', {})
|
||||
self.assertTrue('No connection to' in str(rows))
|
||||
rows = query_member(None, None, None, None, 'SELECT pg_is_in_recovery()')
|
||||
assert 'No connection to' in str(rows)
|
||||
|
||||
rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()', {})
|
||||
self.assertTrue('No connection to' in str(rows))
|
||||
rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
|
||||
assert 'No connection to' in str(rows)
|
||||
|
||||
with patch('patroni.ctl.get_cursor', Mock(side_effect=OperationalError('bla'))):
|
||||
rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()', {})
|
||||
with patch('patroni.ctl.get_cursor', Mock(side_effect=psycopg2.OperationalError('bla'))):
|
||||
rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
|
||||
|
||||
@patch('patroni.ctl.get_dcs')
|
||||
def test_dsn(self, mock_get_dcs):
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
|
||||
result = self.runner.invoke(ctl, ['dsn', 'alpha'])
|
||||
assert 'host=127.0.0.1 port=5435' in result.output
|
||||
with patch('test_postgresql.MockCursor.execute', Mock(side_effect=psycopg2.OperationalError('bla'))):
|
||||
rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
|
||||
|
||||
# Mutually exclusive options
|
||||
result = self.runner.invoke(ctl, ['dsn', 'alpha', '--role', 'master', '--member', 'dummy'])
|
||||
assert result.exit_code == 1
|
||||
@patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
|
||||
def test_dsn(self):
|
||||
runner = CliRunner()
|
||||
|
||||
# Non-existing member
|
||||
result = self.runner.invoke(ctl, ['dsn', 'alpha', '--member', 'dummy'])
|
||||
assert result.exit_code == 1
|
||||
with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)):
|
||||
result = runner.invoke(ctl, ['dsn', 'alpha', '--dcs', '8.8.8.8'])
|
||||
assert 'host=127.0.0.1 port=5435' in result.output
|
||||
|
||||
result = runner.invoke(ctl, [
|
||||
'dsn',
|
||||
'alpha',
|
||||
'--role',
|
||||
'master',
|
||||
'--member',
|
||||
'dummy',
|
||||
])
|
||||
assert 'mutually exclusive' in str(result.exception)
|
||||
|
||||
result = runner.invoke(ctl, ['dsn', 'alpha', '--member', 'dummy'])
|
||||
assert 'Can not find' in str(result.exception)
|
||||
|
||||
# result = runner.invoke(ctl, ['dsn', 'alpha', '--dcs', '8.8.8.8', '--role', 'replica'])
|
||||
# assert 'host=127.0.0.1 port=5436' in result.output
|
||||
|
||||
@patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
|
||||
@patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
|
||||
@patch('requests.get', requests_get)
|
||||
@patch('requests.post', requests_get)
|
||||
@patch('patroni.ctl.get_dcs')
|
||||
def test_restart_reinit(self, mock_get_dcs):
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha'], input='y')
|
||||
assert 'restart failed for' in result.output
|
||||
assert result.exit_code == 0
|
||||
def test_restart_reinit(self):
|
||||
runner = CliRunner()
|
||||
|
||||
result = self.runner.invoke(ctl, ['reinit', 'alpha'], input='y')
|
||||
assert result.exit_code == 1
|
||||
result = runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y')
|
||||
result = runner.invoke(ctl, ['reinit', 'alpha', '--dcs', '8.8.8.8'], input='y')
|
||||
|
||||
# Aborted restart
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha'], input='N')
|
||||
assert result.exit_code == 1
|
||||
|
||||
# Not a member
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha', 'dummy', '--any'], input='y')
|
||||
assert result.exit_code == 1
|
||||
result = runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='N')
|
||||
result = runner.invoke(ctl, [
|
||||
'restart',
|
||||
'alpha',
|
||||
'--dcs',
|
||||
'8.8.8.8',
|
||||
'dummy',
|
||||
'--any',
|
||||
], input='y')
|
||||
assert 'not a member' in str(result.exception)
|
||||
|
||||
with patch('requests.post', Mock(return_value=MockResponse())):
|
||||
result = self.runner.invoke(ctl, ['restart', 'alpha'], input='y')
|
||||
assert result.exit_code == 0
|
||||
result = runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y')
|
||||
|
||||
@patch('patroni.ctl.get_dcs')
|
||||
def test_remove(self, mock_get_dcs):
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
|
||||
result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nslave')
|
||||
@patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
|
||||
@patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
|
||||
def test_remove(self):
|
||||
runner = CliRunner()
|
||||
|
||||
result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='alpha\nslave')
|
||||
assert 'Please confirm' in result.output
|
||||
assert 'You are about to remove all' in result.output
|
||||
# Not typing an exact confirmation
|
||||
assert result.exit_code == 1
|
||||
assert 'You did not exactly type' in str(result.exception)
|
||||
|
||||
# master specified does not match master of cluster
|
||||
result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nYes I am aware\nslave')
|
||||
assert result.exit_code == 1
|
||||
result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='''alpha
|
||||
Yes I am aware
|
||||
slave''')
|
||||
assert 'You did not specify the current master of the cluster' in str(result.exception)
|
||||
|
||||
# cluster specified on cmdline does not match verification prompt
|
||||
result = self.runner.invoke(ctl, ['remove', 'alpha'], input='beta\nleader')
|
||||
assert result.exit_code == 1
|
||||
result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='beta\nleader')
|
||||
assert 'Cluster names specified do not match' in str(result.exception)
|
||||
|
||||
result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nYes I am aware\nleader')
|
||||
assert result.exit_code == 0
|
||||
with patch('patroni.etcd.Etcd.get_cluster', get_cluster_initialized_with_leader):
|
||||
result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'],
|
||||
input='''alpha
|
||||
Yes I am aware
|
||||
leader''')
|
||||
assert 'object has no attribute' in str(result.exception)
|
||||
|
||||
@patch('patroni.dcs.AbstractDCS.watch', Mock(return_value=None))
|
||||
@patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
|
||||
with patch('patroni.ctl.get_dcs', Mock(return_value=Mock())):
|
||||
result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'],
|
||||
input='''alpha
|
||||
Yes I am aware
|
||||
leader''')
|
||||
assert 'We have not implemented this for DCS of type' in str(result.exception)
|
||||
|
||||
@patch('patroni.etcd.Etcd.watch', Mock(return_value=None))
|
||||
@patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
|
||||
def test_wait_for_leader(self):
|
||||
self.assertRaises(PatroniCtlException, wait_for_leader, self.e, 0)
|
||||
dcs = self.e
|
||||
self.assertRaises(patroni.exceptions.PatroniCtlException, wait_for_leader, dcs, 0)
|
||||
|
||||
cluster = wait_for_leader(self.e, timeout=2)
|
||||
cluster = wait_for_leader(dcs=dcs, timeout=2)
|
||||
assert cluster.leader.member.name == 'leader'
|
||||
|
||||
@patch('requests.post', Mock(side_effect=requests.exceptions.ConnectionError('foo')))
|
||||
def test_post_patroni(self):
|
||||
member = get_cluster_initialized_with_leader().leader.member
|
||||
self.assertRaises(requests.exceptions.ConnectionError, post_patroni, member, 'dummy', {})
|
||||
|
||||
def test_ctl(self):
|
||||
self.runner.invoke(ctl, ['list'])
|
||||
runner = CliRunner()
|
||||
|
||||
result = self.runner.invoke(ctl, ['--help'])
|
||||
runner.invoke(ctl, ['list'])
|
||||
|
||||
result = runner.invoke(ctl, ['--help'])
|
||||
assert 'Usage:' in result.output
|
||||
|
||||
def test_get_any_member(self):
|
||||
self.assertIsNone(get_any_member(get_cluster_initialized_without_leader(), role='master'))
|
||||
m = get_any_member(get_cluster_initialized_without_leader(), role='master')
|
||||
assert m is None
|
||||
|
||||
m = get_any_member(get_cluster_initialized_with_leader(), role='master')
|
||||
self.assertEquals(m.name, 'leader')
|
||||
assert m.name == 'leader'
|
||||
|
||||
def test_get_all_members(self):
|
||||
self.assertEquals(list(get_all_members(get_cluster_initialized_without_leader(), role='master')), [])
|
||||
r = list(get_all_members(get_cluster_initialized_without_leader(), role='master'))
|
||||
assert len(r) == 0
|
||||
|
||||
r = list(get_all_members(get_cluster_initialized_with_leader(), role='master'))
|
||||
self.assertEquals(len(r), 1)
|
||||
self.assertEquals(r[0].name, 'leader')
|
||||
assert len(r) == 1
|
||||
assert r[0].name == 'leader'
|
||||
|
||||
r = list(get_all_members(get_cluster_initialized_with_leader(), role='replica'))
|
||||
self.assertEquals(len(r), 1)
|
||||
self.assertEquals(r[0].name, 'other')
|
||||
assert len(r) == 1
|
||||
assert r[0].name == 'other'
|
||||
|
||||
self.assertEquals(len(list(get_all_members(get_cluster_initialized_without_leader(), role='replica'))), 2)
|
||||
r = list(get_all_members(get_cluster_initialized_without_leader(), role='replica'))
|
||||
assert len(r) == 2
|
||||
|
||||
@patch('patroni.ctl.get_dcs')
|
||||
def test_members(self, mock_get_dcs):
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
|
||||
result = self.runner.invoke(members, ['alpha'])
|
||||
assert '127.0.0.1' in result.output
|
||||
@patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
|
||||
@patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
|
||||
@patch('requests.get', requests_get)
|
||||
@patch('requests.post', requests_get)
|
||||
def test_members(self):
|
||||
runner = CliRunner()
|
||||
|
||||
result = runner.invoke(members, ['alpha'])
|
||||
assert result.exit_code == 0
|
||||
|
||||
def test_configure(self):
|
||||
result = self.runner.invoke(configure, ['--dcs', 'abc', '-c', 'dummy', '-n', 'bla'])
|
||||
runner = CliRunner()
|
||||
|
||||
result = runner.invoke(configure, [
|
||||
'--dcs',
|
||||
'abc',
|
||||
'-c',
|
||||
'dummy',
|
||||
'-n',
|
||||
'bla',
|
||||
])
|
||||
|
||||
assert result.exit_code == 0
|
||||
|
||||
|
||||
|
||||
+56
-47
@@ -1,17 +1,17 @@
|
||||
import etcd
|
||||
import json
|
||||
import requests
|
||||
import urllib3
|
||||
import socket
|
||||
import unittest
|
||||
|
||||
from dns.exception import DNSException
|
||||
from mock import Mock, patch
|
||||
from patroni.dcs.etcd import AbstractDCS, Client, Cluster, Etcd, EtcdError
|
||||
from patroni.exceptions import DCSError
|
||||
from urllib3.exceptions import ReadTimeoutError
|
||||
from patroni.dcs import Cluster, DCSError, Leader
|
||||
from patroni.etcd import Client, Etcd, EtcdError
|
||||
|
||||
|
||||
class MockResponse(object):
|
||||
class MockResponse:
|
||||
|
||||
def __init__(self):
|
||||
self.status_code = 200
|
||||
@@ -24,17 +24,26 @@ class MockResponse(object):
|
||||
|
||||
@property
|
||||
def data(self):
|
||||
return self.content.encode('utf-8')
|
||||
if self.content == 'TimeoutError':
|
||||
raise urllib3.exceptions.TimeoutError
|
||||
if self.content == 'Exception':
|
||||
raise Exception
|
||||
return self.content
|
||||
|
||||
@property
|
||||
def status(self):
|
||||
return self.status_code
|
||||
|
||||
@staticmethod
|
||||
def getheader(*args):
|
||||
return ''
|
||||
|
||||
|
||||
class MockPostgresql(Mock):
|
||||
|
||||
def last_operation(self):
|
||||
return '0'
|
||||
|
||||
|
||||
def requests_get(url, **kwargs):
|
||||
members = '[{"id":14855829450254237642,"peerURLs":["http://localhost:2380","http://localhost:7001"],' +\
|
||||
'"name":"default","clientURLs":["http://localhost:2379","http://localhost:4001"]}]'
|
||||
@@ -44,7 +53,10 @@ def requests_get(url, **kwargs):
|
||||
elif ':8011/patroni' in url:
|
||||
response.content = '{"role": "replica", "xlog": {"replayed_location": 0}, "tags": {}}'
|
||||
elif url.endswith('/members'):
|
||||
response.content = '[{}]' if url.startswith('http://error') else members
|
||||
if url.startswith('http://error'):
|
||||
response.content = '[{}]'
|
||||
else:
|
||||
response.content = members
|
||||
elif url.startswith('http://exhibitor'):
|
||||
response.content = '{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}'
|
||||
else:
|
||||
@@ -53,33 +65,35 @@ def requests_get(url, **kwargs):
|
||||
return response
|
||||
|
||||
|
||||
def etcd_watch(self, key, index=None, timeout=None, recursive=None):
|
||||
def etcd_watch(key, index=None, timeout=None, recursive=None):
|
||||
if timeout == 2.0:
|
||||
raise etcd.EtcdWatchTimedOut
|
||||
raise urllib3.exceptions.TimeoutError
|
||||
elif timeout == 5.0:
|
||||
return etcd.EtcdResult('delete', {})
|
||||
elif timeout == 10.0:
|
||||
raise etcd.EtcdException
|
||||
elif index == 20729:
|
||||
return etcd.EtcdResult('set', {'value': 'postgresql1', 'modifiedIndex': index + 1})
|
||||
elif index == 20731:
|
||||
return etcd.EtcdResult('set', {'value': 'postgresql2', 'modifiedIndex': index + 1})
|
||||
|
||||
|
||||
def etcd_write(self, key, value, **kwargs):
|
||||
def etcd_write(key, value, **kwargs):
|
||||
if key == '/service/exists/leader':
|
||||
raise etcd.EtcdAlreadyExist
|
||||
if key in ['/service/test/leader', '/patroni/test/leader'] and \
|
||||
(kwargs.get('prevValue') == 'foo' or not kwargs.get('prevExist', True)):
|
||||
return True
|
||||
if key == '/service/test/leader' or key == '/patroni/test/leader':
|
||||
if kwargs.get('prevValue', None) == 'foo' or not kwargs.get('prevExist', True):
|
||||
return True
|
||||
raise etcd.EtcdException
|
||||
|
||||
|
||||
def etcd_read(self, key, **kwargs):
|
||||
def etcd_read(key, **kwargs):
|
||||
if key == '/service/noleader/':
|
||||
raise DCSError('noleader')
|
||||
elif key == '/service/nocluster/':
|
||||
raise etcd.EtcdKeyNotFound
|
||||
|
||||
response = {"action": "get", "node": {"key": "/service/batman5", "dir": True, "nodes": [
|
||||
{"key": "/service/batman5/config", "value": '{"foo": "bar"}',
|
||||
"modifiedIndex": 1582, "createdIndex": 1582},
|
||||
{"key": "/service/batman5/failover", "value": "",
|
||||
"modifiedIndex": 1582, "createdIndex": 1582},
|
||||
{"key": "/service/batman5/initialize", "value": "postgresql0",
|
||||
@@ -110,12 +124,12 @@ class SleepException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class MockSRV(object):
|
||||
class MockSRV:
|
||||
port = 2380
|
||||
target = '127.0.0.1'
|
||||
|
||||
|
||||
def dns_query(name, _):
|
||||
def dns_query(name, type):
|
||||
if name == '_etcd-server._tcp.blabla':
|
||||
return []
|
||||
elif name == '_etcd-server._tcp.exception':
|
||||
@@ -130,8 +144,6 @@ def socket_getaddrinfo(*args):
|
||||
|
||||
|
||||
def http_request(method, url, **kwargs):
|
||||
if url == 'http://localhost:2379/timeout':
|
||||
raise ReadTimeoutError(None, None, None)
|
||||
if url == 'http://localhost:2379/':
|
||||
return MockResponse()
|
||||
raise socket.error
|
||||
@@ -147,29 +159,32 @@ class TestClient(unittest.TestCase):
|
||||
def setUp(self):
|
||||
with patch.object(etcd.Client, 'machines') as mock_machines:
|
||||
mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
|
||||
self.client = Client({'discovery_srv': 'test', 'retry_timeout': 3})
|
||||
self.client = Client({'discovery_srv': 'test'})
|
||||
self.client.http.request = http_request
|
||||
self.client.http.request_encode_body = http_request
|
||||
|
||||
def test_api_execute(self):
|
||||
self.client._base_uri = 'http://localhost:4001'
|
||||
self.client._machines_cache = ['http://localhost:2379']
|
||||
self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
|
||||
self.client._update_machines_cache = False
|
||||
self.client.api_execute('/', 'POST', timeout=0)
|
||||
self.client.api_execute('/', 'GET')
|
||||
self.client._update_machines_cache = False
|
||||
self.client._base_uri = 'http://localhost:4001'
|
||||
self.client._machines_cache = []
|
||||
self.assertRaises(etcd.EtcdConnectionFailed, self.client.api_execute, '/', 'GET')
|
||||
self.assertTrue(self.client._update_machines_cache)
|
||||
self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'GET')
|
||||
self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
|
||||
self.assertRaises(ValueError, self.client.api_execute, '', '')
|
||||
|
||||
def test_get_srv_record(self):
|
||||
self.assertEquals(self.client.get_srv_record('blabla'), [])
|
||||
self.assertEquals(self.client.get_srv_record('exception'), [])
|
||||
|
||||
def test__result_from_response(self):
|
||||
response = MockResponse()
|
||||
response.content = 'TimeoutError'
|
||||
self.assertRaises(urllib3.exceptions.TimeoutError, self.client._result_from_response, response)
|
||||
response.content = 'Exception'
|
||||
self.assertRaises(etcd.EtcdException, self.client._result_from_response, response)
|
||||
response.content = b'{}'
|
||||
self.assertRaises(etcd.EtcdException, self.client._result_from_response, response)
|
||||
|
||||
def test__get_machines_cache_from_srv(self):
|
||||
self.client.get_srv_record = Mock(return_value=[('localhost', 2380)])
|
||||
self.client._get_machines_cache_from_srv('blabla')
|
||||
@@ -185,16 +200,15 @@ class TestClient(unittest.TestCase):
|
||||
|
||||
|
||||
@patch('requests.get', requests_get)
|
||||
@patch.object(etcd.Client, 'write', etcd_write)
|
||||
@patch.object(etcd.Client, 'read', etcd_read)
|
||||
@patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
|
||||
class TestEtcd(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
with patch.object(Client, 'machines') as mock_machines:
|
||||
mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
|
||||
self.etcd = Etcd({'namespace': '/patroni/', 'ttl': 30, 'retry_timeout': 10,
|
||||
'host': 'localhost:2379', 'scope': 'test', 'name': 'foo'})
|
||||
self.etcd = Etcd('foo', {'namespace': '/patroni/', 'ttl': 30, 'host': 'localhost:2379', 'scope': 'test'})
|
||||
self.etcd.client.write = etcd_write
|
||||
self.etcd.client.read = etcd_read
|
||||
self.etcd.client.delete = Mock(side_effect=etcd.EtcdException())
|
||||
|
||||
def test_base_path(self):
|
||||
self.assertEquals(self.etcd._base_path, '/patroni/test')
|
||||
@@ -204,8 +218,7 @@ class TestEtcd(unittest.TestCase):
|
||||
with patch.object(etcd.Client, 'machines') as mock_machines:
|
||||
mock_machines.__get__ = Mock(side_effect=etcd.EtcdException)
|
||||
with patch('time.sleep', Mock(side_effect=SleepException())):
|
||||
self.assertRaises(SleepException, self.etcd.get_etcd_client,
|
||||
{'discovery_srv': 'test', 'retry_timeout': 10})
|
||||
self.assertRaises(SleepException, self.etcd.get_etcd_client, {'discovery_srv': 'test'})
|
||||
|
||||
def test_get_cluster(self):
|
||||
self.assertIsInstance(self.etcd.get_cluster(), Cluster)
|
||||
@@ -213,8 +226,11 @@ class TestEtcd(unittest.TestCase):
|
||||
cluster = self.etcd.get_cluster()
|
||||
self.assertIsInstance(cluster, Cluster)
|
||||
self.assertIsNone(cluster.leader)
|
||||
|
||||
def test_current_leader(self):
|
||||
self.assertIsInstance(self.etcd.current_leader(), Leader)
|
||||
self.etcd._base_path = '/service/noleader'
|
||||
self.assertRaises(EtcdError, self.etcd.get_cluster)
|
||||
self.assertIsNone(self.etcd.current_leader())
|
||||
|
||||
def test_touch_member(self):
|
||||
self.assertFalse(self.etcd.touch_member('', ''))
|
||||
@@ -243,22 +259,15 @@ class TestEtcd(unittest.TestCase):
|
||||
def test_delete_leader(self):
|
||||
self.assertFalse(self.etcd.delete_leader())
|
||||
|
||||
def test_delete_cluster(self):
|
||||
self.assertFalse(self.etcd.delete_cluster())
|
||||
|
||||
@patch.object(etcd.Client, 'watch', etcd_watch)
|
||||
def test_watch(self):
|
||||
self.etcd.client.watch = etcd_watch
|
||||
self.etcd.watch(0)
|
||||
self.etcd.get_cluster()
|
||||
self.etcd.watch(1.5)
|
||||
self.etcd.watch(4.5)
|
||||
with patch.object(AbstractDCS, 'watch', Mock()):
|
||||
self.etcd.watch(9.5)
|
||||
self.etcd.watch(9.5)
|
||||
self.etcd.watch(100)
|
||||
|
||||
@patch('patroni.etcd.Etcd.retry', Mock(side_effect=AttributeError("foo")))
|
||||
def test_other_exceptions(self):
|
||||
self.etcd.retry = Mock(side_effect=AttributeError('foo'))
|
||||
self.assertRaises(EtcdError, self.etcd.cancel_initialization)
|
||||
|
||||
def test_set_ttl(self):
|
||||
self.etcd.set_ttl(20)
|
||||
self.assertTrue(self.etcd.watch(1))
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
import unittest
|
||||
|
||||
from mock import Mock, patch
|
||||
from patroni.dcs.exhibitor import ExhibitorEnsembleProvider, Exhibitor
|
||||
from patroni.dcs.zookeeper import ZooKeeperError
|
||||
from test_etcd import SleepException, requests_get
|
||||
from test_zookeeper import MockKazooClient
|
||||
|
||||
|
||||
@patch('requests.get', requests_get)
|
||||
@patch('time.sleep', Mock(side_effect=SleepException))
|
||||
class TestExhibitorEnsembleProvider(unittest.TestCase):
|
||||
|
||||
def test_init(self):
|
||||
self.assertRaises(SleepException, ExhibitorEnsembleProvider, ['localhost'], 8181)
|
||||
|
||||
def test_poll(self):
|
||||
self.assertFalse(ExhibitorEnsembleProvider(['exhibitor'], 8181).poll())
|
||||
|
||||
|
||||
class TestExhibitor(unittest.TestCase):
|
||||
|
||||
@patch('requests.get', requests_get)
|
||||
@patch('patroni.dcs.zookeeper.KazooClient', MockKazooClient)
|
||||
def setUp(self):
|
||||
self.e = Exhibitor({'hosts': ['localhost', 'exhibitor'], 'port': 8181, 'scope': 'test',
|
||||
'name': 'foo', 'ttl': 30, 'retry_timeout': 10})
|
||||
|
||||
@patch.object(ExhibitorEnsembleProvider, 'poll', Mock(return_value=True))
|
||||
def test_get_cluster(self):
|
||||
self.assertRaises(ZooKeeperError, self.e.get_cluster)
|
||||
+84
-145
@@ -1,15 +1,11 @@
|
||||
import datetime
|
||||
import etcd
|
||||
import os
|
||||
import pytz
|
||||
import unittest
|
||||
|
||||
from mock import Mock, MagicMock, patch
|
||||
from patroni.config import Config
|
||||
from patroni.dcs import Cluster, Failover, Leader, Member, get_dcs
|
||||
from patroni.dcs import Cluster, Failover, Leader, Member
|
||||
from patroni.etcd import Client, Etcd
|
||||
from patroni.exceptions import DCSError, PostgresException
|
||||
from patroni.ha import Ha
|
||||
from patroni.postgresql import Postgresql
|
||||
from test_etcd import socket_getaddrinfo, etcd_read, etcd_write, requests_get
|
||||
|
||||
|
||||
@@ -22,7 +18,7 @@ def false(*args, **kwargs):
|
||||
|
||||
|
||||
def get_cluster(initialize, leader, members, failover):
|
||||
return Cluster(initialize, None, leader, 10, members, failover)
|
||||
return Cluster(initialize, leader, None, members, failover)
|
||||
|
||||
|
||||
def get_cluster_not_initialized_without_leader():
|
||||
@@ -30,102 +26,92 @@ def get_cluster_not_initialized_without_leader():
|
||||
|
||||
|
||||
def get_cluster_initialized_without_leader(leader=False, failover=None):
|
||||
m1 = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres',
|
||||
'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location': 4})
|
||||
l = Leader(0, 0, m1) if leader else None
|
||||
m2 = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
|
||||
'api_url': 'http://127.0.0.1:8011/patroni', 'tags': {'clonefrom': True}})
|
||||
return get_cluster(True, l, [m1, m2], failover)
|
||||
m = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres',
|
||||
'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location':4})
|
||||
l = Leader(0, 0, m) if leader else None
|
||||
o = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
|
||||
'api_url': 'http://127.0.0.1:8011/patroni'})
|
||||
return get_cluster(True, l, [m, o], failover)
|
||||
|
||||
|
||||
def get_cluster_initialized_with_leader(failover=None):
|
||||
return get_cluster_initialized_without_leader(leader=True, failover=failover)
|
||||
|
||||
|
||||
def get_cluster_initialized_with_only_leader(failover=None):
|
||||
l = get_cluster_initialized_without_leader(leader=True, failover=failover).leader
|
||||
return get_cluster(True, l, [l], failover)
|
||||
|
||||
|
||||
class MockPatroni(object):
|
||||
class MockPostgresql(Mock):
|
||||
|
||||
name = 'postgresql0'
|
||||
role = 'replica'
|
||||
state = 'running'
|
||||
connection_string = 'postgres://foo@bar/postgres'
|
||||
|
||||
def is_healthy(self):
|
||||
return True
|
||||
|
||||
def start(self):
|
||||
return True
|
||||
|
||||
def is_healthiest_node(self, members):
|
||||
return True
|
||||
|
||||
def is_leader(self):
|
||||
return True
|
||||
|
||||
def xlog_position(self):
|
||||
return 0
|
||||
|
||||
def last_operation(self):
|
||||
return 0
|
||||
|
||||
def data_directory_empty(self):
|
||||
return False
|
||||
|
||||
def bootstrap(self, *args, **kwargs):
|
||||
return True
|
||||
|
||||
def check_replication_lag(self, last_leader_operation):
|
||||
return True
|
||||
|
||||
def check_recovery_conf(self, leader):
|
||||
return False
|
||||
|
||||
|
||||
class MockPatroni:
|
||||
|
||||
def __init__(self, p, d):
|
||||
os.environ[Config.PATRONI_CONFIG_VARIABLE] = """
|
||||
restapi:
|
||||
listen: 0.0.0.0:8008
|
||||
bootstrap:
|
||||
users:
|
||||
replicator:
|
||||
password: rep-pass
|
||||
options:
|
||||
- replication
|
||||
postgresql:
|
||||
name: foo
|
||||
data_dir: data/postgresql0
|
||||
pg_rewind:
|
||||
username: postgres
|
||||
password: postgres
|
||||
zookeeper:
|
||||
exhibitor:
|
||||
hosts: [localhost]
|
||||
port: 8181
|
||||
"""
|
||||
self.config = Config()
|
||||
self.postgresql = p
|
||||
self.dcs = d
|
||||
self.api = Mock()
|
||||
self.tags = {'foo': 'bar'}
|
||||
self.tags = {}
|
||||
self.nofailover = None
|
||||
self.nap_time = 10
|
||||
self.replicatefrom = None
|
||||
self.bdr = {}
|
||||
self.api.connection_string = 'http://127.0.0.1:8008'
|
||||
self.clonefrom = None
|
||||
|
||||
|
||||
def run_async(func, args=()):
|
||||
return func(*args) if args else func()
|
||||
func(*args) if args else func()
|
||||
|
||||
|
||||
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
|
||||
@patch.object(Postgresql, 'is_leader', Mock(return_value=True))
|
||||
@patch.object(Postgresql, 'xlog_position', Mock(return_value=0))
|
||||
@patch.object(Postgresql, 'call_nowait', Mock(return_value=True))
|
||||
@patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
|
||||
@patch.object(Postgresql, 'controldata', Mock(return_value={'Database system identifier': '1234567890'}))
|
||||
@patch.object(Postgresql, 'sync_replication_slots', Mock())
|
||||
@patch.object(Postgresql, 'write_pg_hba', Mock())
|
||||
@patch.object(Postgresql, 'write_pgpass', Mock())
|
||||
@patch.object(Postgresql, 'write_recovery_conf', Mock())
|
||||
@patch.object(Postgresql, 'query', Mock())
|
||||
@patch.object(Postgresql, 'checkpoint', Mock())
|
||||
@patch.object(etcd.Client, 'write', etcd_write)
|
||||
@patch.object(etcd.Client, 'read', etcd_read)
|
||||
@patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
|
||||
@patch('subprocess.call', Mock(return_value=0))
|
||||
class TestHa(unittest.TestCase):
|
||||
|
||||
@patch('socket.getaddrinfo', socket_getaddrinfo)
|
||||
@patch.object(etcd.Client, 'read', etcd_read)
|
||||
def setUp(self):
|
||||
with patch.object(etcd.Client, 'machines') as mock_machines:
|
||||
mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
|
||||
self.p = Postgresql({'name': 'postgresql0', 'scope': 'dummy', 'listen': '127.0.0.1:5432',
|
||||
'data_dir': 'data/postgresql0', 'retry_timeout': 10,
|
||||
'authentication': {'superuser': {'username': 'foo', 'password': 'bar'},
|
||||
'replication': {'username': '', 'password': ''}},
|
||||
'parameters': {'wal_level': 'hot_standby', 'max_replication_slots': 5, 'foo': 'bar',
|
||||
'hot_standby': 'on', 'max_wal_senders': 5, 'wal_keep_segments': 8}})
|
||||
self.p.set_state('running')
|
||||
self.p.set_role('replica')
|
||||
self.p.check_replication_lag = true
|
||||
self.p.can_create_replica_without_replication_connection = MagicMock(return_value=False)
|
||||
self.e = get_dcs({'etcd': {'ttl': 30, 'host': 'ok:2379', 'scope': 'test',
|
||||
'name': 'foo', 'retry_timeout': 10}})
|
||||
self.ha = Ha(MockPatroni(self.p, self.e))
|
||||
self.ha._async_executor.run_async = run_async
|
||||
self.ha.old_cluster = self.e.get_cluster()
|
||||
self.ha.cluster = get_cluster_not_initialized_without_leader()
|
||||
self.ha.load_cluster_from_dcs = Mock()
|
||||
@patch.object(Client, 'machines')
|
||||
def setUp(self, mock_machines):
|
||||
mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
|
||||
self.p = MockPostgresql()
|
||||
self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
|
||||
self.e.client.read = etcd_read
|
||||
self.e.client.write = etcd_write
|
||||
self.e.client.delete = Mock(side_effect=etcd.EtcdException())
|
||||
self.ha = Ha(MockPatroni(self.p, self.e))
|
||||
self.ha._async_executor.run_async = run_async
|
||||
self.ha.old_cluster = self.e.get_cluster()
|
||||
self.ha.cluster = get_cluster_not_initialized_without_leader()
|
||||
self.ha.load_cluster_from_dcs = Mock()
|
||||
|
||||
def test_update_lock(self):
|
||||
self.p.last_operation = Mock(side_effect=PostgresException(''))
|
||||
@@ -137,25 +123,19 @@ class TestHa(unittest.TestCase):
|
||||
|
||||
def test_start_as_replica(self):
|
||||
self.p.is_healthy = false
|
||||
self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
|
||||
self.assertEquals(self.ha.run_cycle(), 'started as a secondary')
|
||||
|
||||
def test_recover_replica_failed(self):
|
||||
self.p.controldata = lambda: {'Database cluster state': 'in production'}
|
||||
self.p.is_healthy = false
|
||||
self.p.is_running = false
|
||||
self.p.follow = false
|
||||
self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
|
||||
self.p.follow_the_leader = false
|
||||
self.assertEquals(self.ha.run_cycle(), 'failed to start postgres')
|
||||
|
||||
def test_recover_master_failed(self):
|
||||
self.p.follow = false
|
||||
self.p.follow_the_leader = false
|
||||
self.p.is_healthy = false
|
||||
self.p.is_running = false
|
||||
self.p.name = 'leader'
|
||||
self.p.set_role('master')
|
||||
self.p.controldata = lambda: {'Database cluster state': 'in production'}
|
||||
self.ha.cluster = get_cluster_initialized_with_leader()
|
||||
self.assertEquals(self.ha.run_cycle(), 'starting as readonly because i had the session lock')
|
||||
self.ha.has_lock = true
|
||||
self.assertEquals(self.ha.run_cycle(), 'removed leader key after trying and failing to start postgres')
|
||||
|
||||
@patch('sys.exit', return_value=1)
|
||||
@patch('patroni.ha.Ha.sysid_valid', MagicMock(return_value=True))
|
||||
@@ -165,8 +145,7 @@ class TestHa(unittest.TestCase):
|
||||
|
||||
@patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
|
||||
def test_start_as_readonly(self):
|
||||
self.p.is_leader = false
|
||||
self.p.is_healthy = true
|
||||
self.p.is_leader = self.p.is_healthy = false
|
||||
self.ha.has_lock = true
|
||||
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader because i had the session lock')
|
||||
|
||||
@@ -180,7 +159,7 @@ class TestHa(unittest.TestCase):
|
||||
|
||||
def test_demote_after_failing_to_obtain_lock(self):
|
||||
self.ha.acquire_lock = false
|
||||
self.assertEquals(self.ha.run_cycle(), 'demoted self after trying and failing to obtain lock')
|
||||
self.assertEquals(self.ha.run_cycle(), 'demoted self due after trying and failing to obtain lock')
|
||||
|
||||
def test_follow_new_leader_after_failing_to_obtain_lock(self):
|
||||
self.ha.is_healthiest_node = true
|
||||
@@ -218,30 +197,24 @@ class TestHa(unittest.TestCase):
|
||||
self.ha.update_lock = false
|
||||
self.assertEquals(self.ha.run_cycle(), 'demoting self because i do not have the lock and i was a leader')
|
||||
|
||||
def test_follow(self):
|
||||
def test_follow_the_leader(self):
|
||||
self.ha.cluster.is_unlocked = false
|
||||
self.p.is_leader = false
|
||||
self.assertEquals(self.ha.run_cycle(), 'no action. i am a secondary and i am following a leader')
|
||||
self.ha.patroni.replicatefrom = "foo"
|
||||
self.assertEquals(self.ha.run_cycle(), 'no action. i am a secondary and i am following a leader')
|
||||
|
||||
def test_no_etcd_connection_master_demote(self):
|
||||
self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
|
||||
self.assertEquals(self.ha.run_cycle(), 'demoted self because DCS is not accessible and i was a leader')
|
||||
|
||||
def test_bootstrap_from_another_member(self):
|
||||
def test_bootstrap_from_leader(self):
|
||||
self.ha.cluster = get_cluster_initialized_with_leader()
|
||||
self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap from replica \'other\'')
|
||||
self.p.bootstrap = false
|
||||
self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap from leader')
|
||||
|
||||
def test_bootstrap_waiting_for_leader(self):
|
||||
self.ha.cluster = get_cluster_initialized_without_leader()
|
||||
self.assertEquals(self.ha.bootstrap(), 'waiting for leader to bootstrap')
|
||||
|
||||
def test_bootstrap_without_leader(self):
|
||||
self.ha.cluster = get_cluster_initialized_without_leader()
|
||||
self.p.can_create_replica_without_replication_connection = MagicMock(return_value=True)
|
||||
self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap (without leader)')
|
||||
|
||||
def test_bootstrap_initialize_lock_failed(self):
|
||||
self.ha.cluster = get_cluster_not_initialized_without_leader()
|
||||
self.assertEquals(self.ha.bootstrap(), 'failed to acquire initialize lock')
|
||||
@@ -295,65 +268,40 @@ class TestHa(unittest.TestCase):
|
||||
self.assertEquals(self.ha.run_cycle(), 'failed to update leader lock during restart')
|
||||
|
||||
@patch('requests.get', requests_get)
|
||||
@patch('time.sleep', Mock())
|
||||
def test_manual_failover_from_leader(self):
|
||||
self.ha.has_lock = true
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None))
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', ''))
|
||||
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None))
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', MockPostgresql.name))
|
||||
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla', None))
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla'))
|
||||
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
|
||||
f = Failover(0, self.p.name, '', None)
|
||||
f = Failover(0, MockPostgresql.name, '')
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(f)
|
||||
self.assertEquals(self.ha.run_cycle(), 'manual failover: demoting myself')
|
||||
self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'})
|
||||
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
|
||||
# manual failover from the previous leader to us won't happen if we hold the nofailover flag
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, None))
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name))
|
||||
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
|
||||
|
||||
# Failover scheduled time must include timezone
|
||||
scheduled = datetime.datetime.now()
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.ha.run_cycle()
|
||||
|
||||
scheduled = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle())
|
||||
|
||||
scheduled = scheduled + datetime.timedelta(seconds=30)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle())
|
||||
|
||||
scheduled = scheduled + datetime.timedelta(seconds=-600)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle())
|
||||
|
||||
scheduled = None
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle())
|
||||
|
||||
@patch('requests.get', requests_get)
|
||||
def test_manual_failover_process_no_leader(self):
|
||||
self.p.is_leader = false
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', self.p.name, None))
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', MockPostgresql.name))
|
||||
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None))
|
||||
self.p.set_role('replica')
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader'))
|
||||
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
self.ha.fetch_node_status = lambda e: (e, True, True, 0, {}) # accessible, in_recovery
|
||||
self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, self.p.name, '', None))
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, MockPostgresql.name, ''))
|
||||
self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
self.ha.fetch_node_status = lambda e: (e, False, True, 0, {}) # inaccessible, in_recovery
|
||||
self.p.set_role('replica')
|
||||
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
# set failover flag to True for all members of the cluster
|
||||
# this should elect the current member, as we are not going to call the API for it.
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other'))
|
||||
self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'}) # accessible, in_recovery
|
||||
self.p.set_role('replica')
|
||||
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
# same as previous, but set the current member to nofailover. In no case it should be elected as a leader
|
||||
self.ha.patroni.nofailover = True
|
||||
@@ -386,12 +334,3 @@ class TestHa(unittest.TestCase):
|
||||
self.ha.fetch_node_status(member)
|
||||
member = Member(0, 'test', 1, {'api_url': 'http://localhost:8011/patroni'})
|
||||
self.ha.fetch_node_status(member)
|
||||
|
||||
def test_post_recover(self):
|
||||
self.p.is_running = false
|
||||
self.ha.has_lock = true
|
||||
self.assertEqual(self.ha.post_recover(), 'removed leader key after trying and failing to start postgres')
|
||||
self.ha.has_lock = false
|
||||
self.assertEqual(self.ha.post_recover(), 'failed to start postgres')
|
||||
self.p.is_running = true
|
||||
self.assertIsNone(self.ha.post_recover())
|
||||
|
||||
+39
-55
@@ -1,75 +1,73 @@
|
||||
import etcd
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
import yaml
|
||||
|
||||
from mock import Mock, patch
|
||||
from patroni.api import RestApiServer
|
||||
from patroni.async_executor import AsyncExecutor
|
||||
from patroni.exceptions import DCSError
|
||||
from patroni import Patroni, main as _main
|
||||
from patroni.etcd import Etcd
|
||||
from patroni import Patroni, main
|
||||
from patroni.zookeeper import ZooKeeper
|
||||
from six.moves import BaseHTTPServer
|
||||
from test_etcd import SleepException, etcd_read, etcd_write
|
||||
from test_etcd import Client, SleepException, etcd_read, etcd_write
|
||||
from test_postgresql import Postgresql, psycopg2_connect
|
||||
from test_zookeeper import MockKazooClient
|
||||
|
||||
|
||||
def time_sleep(*args):
|
||||
raise SleepException()
|
||||
|
||||
|
||||
@patch('time.sleep', Mock())
|
||||
@patch('subprocess.call', Mock(return_value=0))
|
||||
@patch('psycopg2.connect', psycopg2_connect)
|
||||
@patch.object(Postgresql, 'write_pg_hba', Mock())
|
||||
@patch.object(Postgresql, '_write_postgresql_conf', Mock())
|
||||
@patch.object(Postgresql, 'write_recovery_conf', Mock())
|
||||
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
|
||||
@patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
|
||||
@patch.object(AsyncExecutor, 'run', Mock())
|
||||
@patch.object(etcd.Client, 'write', etcd_write)
|
||||
@patch.object(etcd.Client, 'read', etcd_read)
|
||||
class TestPatroni(unittest.TestCase):
|
||||
|
||||
@patch.object(etcd.Client, 'read', etcd_read)
|
||||
def setUp(self):
|
||||
@patch.object(Client, 'machines')
|
||||
def setUp(self, mock_machines):
|
||||
mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
|
||||
self.touched = False
|
||||
self.init_cancelled = False
|
||||
RestApiServer._BaseServer__is_shut_down = Mock()
|
||||
RestApiServer._BaseServer__shutdown_request = True
|
||||
RestApiServer.socket = 0
|
||||
with patch.object(etcd.Client, 'machines') as mock_machines:
|
||||
mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
|
||||
sys.argv = ['patroni.py', 'postgres0.yml']
|
||||
self.p = Patroni()
|
||||
with open('postgres0.yml', 'r') as f:
|
||||
config = yaml.load(f)
|
||||
self.p = Patroni(config)
|
||||
self.p.ha.dcs.client.write = etcd_write
|
||||
self.p.ha.dcs.client.read = etcd_read
|
||||
|
||||
@patch('patroni.dcs.AbstractDCS.get_cluster', Mock(side_effect=[None, DCSError('foo'), None]))
|
||||
def test_load_dynamic_configuration(self):
|
||||
self.p.config._dynamic_configuration = {}
|
||||
self.p.load_dynamic_configuration()
|
||||
self.p.load_dynamic_configuration()
|
||||
@patch('patroni.zookeeper.KazooClient', MockKazooClient())
|
||||
def test_get_dcs(self):
|
||||
self.assertIsInstance(self.p.get_dcs('', {'zookeeper': {'scope': '', 'hosts': ''}}), ZooKeeper)
|
||||
self.assertRaises(Exception, self.p.get_dcs, '', {})
|
||||
|
||||
@patch('time.sleep', Mock(side_effect=SleepException))
|
||||
@patch.object(etcd.Client, 'delete', Mock())
|
||||
@patch.object(etcd.Client, 'machines')
|
||||
@patch('time.sleep', Mock(side_effect=SleepException()))
|
||||
@patch.object(Etcd, 'delete_leader', Mock())
|
||||
@patch.object(Client, 'machines')
|
||||
def test_patroni_main(self, mock_machines):
|
||||
with patch('subprocess.call', Mock(return_value=1)):
|
||||
sys.argv = ['patroni.py', 'postgres0.yml']
|
||||
main()
|
||||
sys.argv = ['patroni.py', 'postgres0.yml']
|
||||
|
||||
mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
|
||||
with patch.object(Patroni, 'run', Mock(side_effect=SleepException)):
|
||||
self.assertRaises(SleepException, _main)
|
||||
with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())):
|
||||
_main()
|
||||
mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
|
||||
with patch.object(Patroni, 'run', Mock(side_effect=SleepException())):
|
||||
self.assertRaises(SleepException, main)
|
||||
with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())):
|
||||
main()
|
||||
|
||||
@patch('patroni.config.Config.save_cache', Mock())
|
||||
@patch('patroni.config.Config.reload_local_configuration', Mock(return_value=True))
|
||||
@patch('time.sleep', Mock(side_effect=SleepException()))
|
||||
def test_run(self):
|
||||
self.p.sighup_handler()
|
||||
self.p.ha.dcs.watch = Mock(side_effect=SleepException)
|
||||
self.p.api.start = Mock()
|
||||
self.p.config._dynamic_configuration = {}
|
||||
self.p.ha.dcs.watch = time_sleep
|
||||
self.assertRaises(SleepException, self.p.run)
|
||||
with patch('patroni.config.Config.set_dynamic_configuration', Mock(return_value=True)):
|
||||
self.assertRaises(SleepException, self.p.run)
|
||||
with patch('patroni.postgresql.Postgresql.data_directory_empty', Mock(return_value=False)):
|
||||
self.assertRaises(SleepException, self.p.run)
|
||||
|
||||
def test_sigterm_handler(self):
|
||||
self.assertRaises(SystemExit, self.p.sigterm_handler)
|
||||
self.p.ha.state_handler.is_leader = Mock(return_value=False)
|
||||
self.p.api.start = Mock()
|
||||
self.assertRaises(SleepException, self.p.run)
|
||||
|
||||
def test_schedule_next_run(self):
|
||||
self.p.ha.dcs.watch = Mock(return_value=True)
|
||||
@@ -77,22 +75,8 @@ class TestPatroni(unittest.TestCase):
|
||||
self.p.next_run = time.time() - self.p.nap_time - 1
|
||||
self.p.schedule_next_run()
|
||||
|
||||
def test_noloadbalance(self):
|
||||
self.p.tags['noloadbalance'] = True
|
||||
self.assertTrue(self.p.noloadbalance)
|
||||
|
||||
def test_nofailover(self):
|
||||
self.p.tags['nofailover'] = True
|
||||
self.assertTrue(self.p.nofailover)
|
||||
self.p.tags['nofailover'] = None
|
||||
self.assertFalse(self.p.nofailover)
|
||||
|
||||
def test_replicatefrom(self):
|
||||
self.assertIsNone(self.p.replicatefrom)
|
||||
self.p.tags['replicatefrom'] = 'foo'
|
||||
self.assertEqual(self.p.replicatefrom, 'foo')
|
||||
|
||||
def test_reload_config(self):
|
||||
self.p.reload_config()
|
||||
self.p.get_tags = Mock(side_effect=Exception)
|
||||
self.p.reload_config()
|
||||
|
||||
+169
-226
@@ -2,19 +2,24 @@ import mock # for the mock.call method, importing it without a namespace breaks
|
||||
import os
|
||||
import psycopg2
|
||||
import shutil
|
||||
import subprocess
|
||||
import unittest
|
||||
|
||||
from six.moves import builtins
|
||||
from mock import Mock, MagicMock, PropertyMock, patch, mock_open
|
||||
from patroni.dcs import Cluster, Leader, Member
|
||||
from patroni.exceptions import PostgresException, PostgresConnectionException
|
||||
from patroni.postgresql import Postgresql
|
||||
from patroni.utils import RetryFailedError
|
||||
from six.moves import builtins
|
||||
from test_ha import false
|
||||
import subprocess
|
||||
|
||||
|
||||
class MockCursor(object):
|
||||
def is_file_raise_on_backup(*args, **kwargs):
|
||||
if args[0].endswith('.backup'):
|
||||
raise Exception("foo")
|
||||
|
||||
|
||||
class MockCursor:
|
||||
|
||||
def __init__(self, connection):
|
||||
self.connection = connection
|
||||
@@ -33,15 +38,20 @@ class MockCursor(object):
|
||||
elif sql == 'SELECT pg_is_in_recovery()':
|
||||
self.results = [(False, )]
|
||||
elif sql.startswith('SELECT to_char(pg_postmaster_start_time'):
|
||||
self.results = [('', True, '', '', '', '', False)]
|
||||
elif sql.startswith('SELECT name, setting'):
|
||||
self.results = [('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
|
||||
('search_path', 'public', None, 'string', 'user'),
|
||||
('port', '5433', None, 'integer', 'postmaster'),
|
||||
('listen_addresses', '*', None, 'string', 'postmaster'),
|
||||
('autovacuum', 'on', None, 'bool', 'sighup')]
|
||||
self.results = [('', True, '', '', '', False)]
|
||||
else:
|
||||
self.results = [(None, None, None, None, None, None, None, None, None, None)]
|
||||
self.results = [(
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)]
|
||||
|
||||
def fetchone(self):
|
||||
return self.results[0]
|
||||
@@ -49,6 +59,9 @@ class MockCursor(object):
|
||||
def fetchall(self):
|
||||
return self.results
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
def __iter__(self):
|
||||
for i in self.results:
|
||||
yield i
|
||||
@@ -60,9 +73,8 @@ class MockCursor(object):
|
||||
pass
|
||||
|
||||
|
||||
class MockConnect(object):
|
||||
class MockConnect(Mock):
|
||||
|
||||
server_version = '99999'
|
||||
autocommit = False
|
||||
closed = 0
|
||||
|
||||
@@ -75,10 +87,6 @@ class MockConnect(object):
|
||||
def __exit__(self, *args):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def close():
|
||||
pass
|
||||
|
||||
|
||||
def pg_controldata_string(*args, **kwargs):
|
||||
return b"""
|
||||
@@ -146,84 +154,72 @@ def psycopg2_connect(*args, **kwargs):
|
||||
return MockConnect()
|
||||
|
||||
|
||||
def fake_listdir(path):
|
||||
return ["a", "b", "c"] if path.endswith('pg_xlog/archive_status') else []
|
||||
|
||||
|
||||
@patch('subprocess.call', Mock(return_value=0))
|
||||
@patch('psycopg2.connect', psycopg2_connect)
|
||||
@patch('shutil.copy', Mock())
|
||||
class TestPostgresql(unittest.TestCase):
|
||||
_PARAMETERS = {'wal_level': 'hot_standby', 'max_replication_slots': 5, 'f.oo': 'bar',
|
||||
'search_path': 'public', 'hot_standby': 'on', 'max_wal_senders': 5,
|
||||
'wal_keep_segments': 8, 'wal_log_hints': 'on', 'max_locks_per_transaction': 64,
|
||||
'max_worker_processes': 8, 'max_connections': 100, 'max_prepared_transactions': 0}
|
||||
|
||||
@patch('subprocess.call', Mock(return_value=0))
|
||||
@patch('psycopg2.connect', psycopg2_connect)
|
||||
@patch('os.rename', Mock())
|
||||
@patch.object(Postgresql, 'get_major_version', Mock(return_value=9.4))
|
||||
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
|
||||
def setUp(self):
|
||||
self.data_dir = 'data/test0'
|
||||
if not os.path.exists(self.data_dir):
|
||||
os.makedirs(self.data_dir)
|
||||
self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': self.data_dir, 'retry_timeout': 10,
|
||||
self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': 'data/test0',
|
||||
'listen': '127.0.0.1, *:5432', 'connect_address': '127.0.0.2:5432',
|
||||
'authentication': {'superuser': {'username': 'test', 'password': 'test'},
|
||||
'replication': {'username': 'replicator', 'password': 'rep-pass'}},
|
||||
'remove_data_directory_on_rewind_failure': True,
|
||||
'use_pg_rewind': True, 'pg_ctl_timeout': 'bla',
|
||||
'parameters': self._PARAMETERS,
|
||||
'recovery_conf': {'foo': 'bar'},
|
||||
'pg_hba': ['hostssl all all 0.0.0.0/0 md5', 'host all all 0.0.0.0/0 md5'],
|
||||
'superuser': {'password': 'test'},
|
||||
'admin': {'username': 'admin', 'password': 'admin'},
|
||||
'pg_rewind': {'username': 'admin', 'password': 'admin'},
|
||||
'replication': {'username': 'replicator',
|
||||
'password': 'rep-pass',
|
||||
'network': '127.0.0.1/32'},
|
||||
'parameters': {'foo': 'bar'}, 'recovery_conf': {'foo': 'bar'},
|
||||
'callbacks': {'on_start': 'true', 'on_stop': 'true',
|
||||
'on_restart': 'true', 'on_role_change': 'true',
|
||||
'on_reload': 'true'
|
||||
},
|
||||
'restore': 'true'})
|
||||
if not os.path.exists(self.p.data_dir):
|
||||
os.makedirs(self.p.data_dir)
|
||||
self.leadermem = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres'})
|
||||
self.leader = Leader(-1, 28, self.leadermem)
|
||||
self.other = Member(0, 'test1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres',
|
||||
'tags': {'replicatefrom': 'leader'}})
|
||||
self.other = Member(0, 'test1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres'})
|
||||
self.me = Member(0, 'test0', 28, {'conn_url': 'postgres://replicator:[email protected]:5434/postgres'})
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree('data')
|
||||
|
||||
def test_data_directory_empty(self):
|
||||
self.assertTrue(self.p.data_directory_empty())
|
||||
|
||||
def test_get_initdb_options(self):
|
||||
self.assertEquals(self.p.get_initdb_options([{'encoding': 'UTF8'}, 'data-checksums']),
|
||||
['--encoding=UTF8', '--data-checksums'])
|
||||
self.assertRaises(Exception, self.p.get_initdb_options, [{'pgdata': 'bar'}])
|
||||
self.assertRaises(Exception, self.p.get_initdb_options, [{'foo': 'bar', 1: 2}])
|
||||
self.assertRaises(Exception, self.p.get_initdb_options, [1])
|
||||
self.p.initdb_options = [{'encoding': 'UTF8'}, 'data-checksums']
|
||||
self.assertEquals(self.p.get_initdb_options(), ['--encoding=UTF8', '--data-checksums'])
|
||||
self.p.initdb_options = [{'pgdata': 'bar'}]
|
||||
self.assertRaises(Exception, self.p.get_initdb_options)
|
||||
self.p.initdb_options = [{'foo': 'bar', 1: 2}]
|
||||
self.assertRaises(Exception, self.p.get_initdb_options)
|
||||
self.p.initdb_options = [1]
|
||||
self.assertRaises(Exception, self.p.get_initdb_options)
|
||||
|
||||
@patch('os.path.exists', Mock(return_value=True))
|
||||
@patch('os.unlink', Mock())
|
||||
def test_delete_trigger_file(self):
|
||||
self.p.delete_trigger_file()
|
||||
def test_initialize(self):
|
||||
self.assertTrue(self.p.initialize())
|
||||
self.assertTrue(os.path.exists(os.path.join(self.p.data_dir, 'pg_hba.conf')))
|
||||
|
||||
@patch.object(Postgresql, 'is_running')
|
||||
def test_start(self, mock_is_running):
|
||||
mock_is_running.return_value = True
|
||||
def test_start(self):
|
||||
self.assertTrue(self.p.start())
|
||||
mock_is_running.return_value = False
|
||||
open(os.path.join(self.data_dir, 'postmaster.pid'), 'w').close()
|
||||
pg_conf = os.path.join(self.data_dir, 'postgresql.conf')
|
||||
open(pg_conf, 'w').close()
|
||||
self.p.is_running = false
|
||||
open(os.path.join(self.p.data_dir, 'postmaster.pid'), 'w').close()
|
||||
self.assertTrue(self.p.start())
|
||||
with open(pg_conf) as f:
|
||||
lines = f.readlines()
|
||||
self.assertTrue("f.oo = 'bar'\n" in lines)
|
||||
|
||||
@patch.object(Postgresql, 'is_running')
|
||||
def test_stop(self, mock_is_running):
|
||||
mock_is_running.return_value = True
|
||||
def test_stop(self):
|
||||
self.assertTrue(self.p.stop())
|
||||
with patch('subprocess.call', Mock(return_value=1)):
|
||||
mock_is_running.return_value = False
|
||||
self.assertTrue(self.p.stop())
|
||||
self.p.is_running = Mock(return_value=True)
|
||||
self.assertFalse(self.p.stop())
|
||||
|
||||
def test_restart(self):
|
||||
self.p.start = false
|
||||
self.p.is_running = false
|
||||
self.assertFalse(self.p.restart())
|
||||
self.assertEquals(self.p.state, 'restart failed (restarting)')
|
||||
|
||||
@@ -231,101 +227,92 @@ class TestPostgresql(unittest.TestCase):
|
||||
def test_write_pgpass(self):
|
||||
self.p.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo', 'password': 'bar'})
|
||||
|
||||
def test_checkpoint(self):
|
||||
with patch.object(MockCursor, 'fetchone', Mock(return_value=(True, ))):
|
||||
self.assertEquals(self.p.checkpoint({'user': 'postgres'}), 'is_in_recovery=true')
|
||||
with patch.object(MockCursor, 'execute', Mock(return_value=None)):
|
||||
self.assertIsNone(self.p.checkpoint())
|
||||
self.assertEquals(self.p.checkpoint(), 'not accessible or not healty')
|
||||
@patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
|
||||
def test_sync_from_leader(self):
|
||||
self.assertTrue(self.p.sync_from_leader(self.leader))
|
||||
|
||||
@patch('subprocess.call', side_effect=OSError)
|
||||
@patch('subprocess.call', side_effect=Exception("Test"))
|
||||
@patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
|
||||
def test_pg_rewind(self, mock_call):
|
||||
r = {'user': '', 'host': '', 'port': '', 'database': '', 'password': ''}
|
||||
self.assertTrue(self.p.rewind(r))
|
||||
self.assertTrue(self.p.rewind(self.leader))
|
||||
subprocess.call = mock_call
|
||||
self.assertFalse(self.p.rewind(r))
|
||||
self.assertFalse(self.p.rewind(self.leader))
|
||||
|
||||
@patch('os.unlink', Mock(return_value=True))
|
||||
@patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string))
|
||||
@patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True))
|
||||
@patch.object(Postgresql, 'single_user_mode', Mock(return_value=1))
|
||||
@patch.object(Postgresql, 'write_pgpass', Mock(return_value={}))
|
||||
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
|
||||
@patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
|
||||
@patch.object(Postgresql, 'rewind', return_value=False)
|
||||
def test_follow(self, mock_pg_rewind):
|
||||
with patch.object(Postgresql, 'check_recovery_conf', Mock(return_value=True)):
|
||||
self.assertTrue(self.p.follow(None, None)) # nothing to do, recovery.conf has good primary_conninfo
|
||||
@patch('patroni.postgresql.Postgresql.rewind', return_value=False)
|
||||
@patch('patroni.postgresql.Postgresql.remove_data_directory', MagicMock(return_value=True))
|
||||
@patch('patroni.postgresql.Postgresql.single_user_mode', MagicMock(return_value=1))
|
||||
@patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
|
||||
def test_follow_the_leader(self, mock_pg_rewind):
|
||||
self.p.demote()
|
||||
self.p.follow_the_leader(None)
|
||||
self.p.demote()
|
||||
self.p.follow_the_leader(self.leader)
|
||||
self.p.follow_the_leader(Leader(-1, 28, self.other))
|
||||
self.p.rewind = mock_pg_rewind
|
||||
self.p.follow_the_leader(self.leader)
|
||||
self.p.require_rewind()
|
||||
with mock.patch('os.path.islink', MagicMock(return_value=True)):
|
||||
with mock.patch('patroni.postgresql.Postgresql.can_rewind', new_callable=PropertyMock(return_value=True)):
|
||||
with mock.patch('os.unlink', MagicMock(return_value=True)):
|
||||
self.p.follow_the_leader(self.leader, recovery=True)
|
||||
self.p.require_rewind()
|
||||
with mock.patch('patroni.postgresql.Postgresql.can_rewind', new_callable=PropertyMock(return_value=True)):
|
||||
self.p.rewind.return_value = True
|
||||
self.p.follow_the_leader(self.leader, recovery=True)
|
||||
self.p.rewind.return_value = False
|
||||
self.p.follow_the_leader(self.leader, recovery=True)
|
||||
|
||||
self.p.follow(self.me, self.me) # follow is called when the node is holding leader lock
|
||||
|
||||
with patch.object(Postgresql, 'restart', Mock(return_value=False)):
|
||||
self.p.set_role('replica')
|
||||
self.p.follow(None, None) # restart without rewind
|
||||
self.p.set_role('master')
|
||||
|
||||
with patch.object(Postgresql, 'stop', Mock(return_value=False)):
|
||||
self.p.follow(self.leader, self.leader) # failed to stop postgres
|
||||
|
||||
self.p.follow(self.leader, None) # Leader unknown, can not rewind
|
||||
|
||||
self.p.follow(self.leader, self.leader) # "leader" is not accessible or is_in_recovery
|
||||
|
||||
with patch.object(Postgresql, 'checkpoint', Mock(return_value=None)):
|
||||
self.p.follow(self.leader, self.leader)
|
||||
self.p.set_role('master')
|
||||
mock_pg_rewind.return_value = True
|
||||
self.p.follow(self.leader, self.leader)
|
||||
|
||||
self.assertTrue(self.p.follow(None, None)) # check_recovery_conf...
|
||||
|
||||
@patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string))
|
||||
def test_can_rewind(self):
|
||||
with patch('subprocess.call', MagicMock(return_value=1)):
|
||||
self.assertFalse(self.p.can_rewind)
|
||||
with patch('subprocess.call', side_effect=OSError):
|
||||
self.assertFalse(self.p.can_rewind)
|
||||
with patch.object(Postgresql, 'controldata', Mock(return_value={'wal_log_hints setting': 'on'})):
|
||||
self.assertTrue(self.p.can_rewind)
|
||||
self.p.config['use_pg_rewind'] = False
|
||||
tmp = self.p.pg_rewind
|
||||
self.p.pg_rewind = None
|
||||
self.assertFalse(self.p.can_rewind)
|
||||
self.p.pg_rewind = tmp
|
||||
with mock.patch('subprocess.call', MagicMock(return_value=1)):
|
||||
self.assertFalse(self.p.can_rewind)
|
||||
with mock.patch('subprocess.call', side_effect=OSError("foo")):
|
||||
self.assertFalse(self.p.can_rewind)
|
||||
tmp = self.p.controldata()
|
||||
self.p.controldata = lambda: {'wal_log_hints setting': 'on'}
|
||||
self.assertTrue(self.p.can_rewind)
|
||||
self.p.controldata = tmp
|
||||
|
||||
@patch('time.sleep', Mock())
|
||||
def test_create_replica(self):
|
||||
self.p.delete_trigger_file = Mock(side_effect=OSError)
|
||||
self.p.delete_trigger_file = Mock(side_effect=OSError())
|
||||
with patch('subprocess.call', Mock(side_effect=[1, 0])):
|
||||
self.assertEquals(self.p.create_replica(self.leader), 0)
|
||||
self.assertEquals(self.p.create_replica(self.leader, ''), 0)
|
||||
with patch('subprocess.call', Mock(side_effect=[Exception(), 0])):
|
||||
self.assertEquals(self.p.create_replica(self.leader), 0)
|
||||
self.assertEquals(self.p.create_replica(self.leader, ''), 0)
|
||||
|
||||
self.p.config['create_replica_method'] = ['wale', 'basebackup']
|
||||
self.p.config['wale'] = {'command': 'foo'}
|
||||
with patch('subprocess.call', Mock(return_value=0)):
|
||||
self.assertEquals(self.p.create_replica(self.leader), 0)
|
||||
self.assertEquals(self.p.create_replica(self.leader, ''), 0)
|
||||
del self.p.config['wale']
|
||||
self.assertEquals(self.p.create_replica(self.leader), 0)
|
||||
self.assertEquals(self.p.create_replica(self.leader, ''), 0)
|
||||
|
||||
with patch('subprocess.call', Mock(side_effect=Exception("foo"))):
|
||||
self.assertEquals(self.p.create_replica(self.leader), 1)
|
||||
self.assertEquals(self.p.create_replica(self.leader, ''), 1)
|
||||
|
||||
def test_create_connection_users(self):
|
||||
cfg = self.p.config
|
||||
cfg['superuser']['username'] = 'test'
|
||||
p = Postgresql(cfg)
|
||||
p.create_connection_users()
|
||||
|
||||
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
|
||||
def test_sync_replication_slots(self):
|
||||
self.p.start()
|
||||
cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None)
|
||||
cluster = Cluster(True, self.leader, 0, [self.me, self.other, self.leadermem], None)
|
||||
self.p.sync_replication_slots(cluster)
|
||||
self.p.query = Mock(side_effect=psycopg2.OperationalError)
|
||||
self.p.schedule_load_slots = True
|
||||
self.p.sync_replication_slots(cluster)
|
||||
self.p.schedule_load_slots = False
|
||||
with mock.patch('patroni.postgresql.Postgresql.role', new_callable=PropertyMock(return_value='replica')):
|
||||
self.p.sync_replication_slots(cluster)
|
||||
|
||||
@patch.object(MockConnect, 'closed', 2)
|
||||
def test__query(self):
|
||||
self.assertRaises(PostgresConnectionException, self.p._query, 'blabla')
|
||||
self.assertRaises(PostgresConnectionException, self.p._query, 'blabla', 'postgres')
|
||||
self.p._state = 'restarting'
|
||||
self.assertRaises(RetryFailedError, self.p._query, 'blabla')
|
||||
self.assertRaises(RetryFailedError, self.p._query, 'blabla', 'postgres')
|
||||
|
||||
def test_query(self):
|
||||
self.p.query('select 1')
|
||||
@@ -338,11 +325,9 @@ class TestPostgresql(unittest.TestCase):
|
||||
def test_reload(self):
|
||||
self.assertTrue(self.p.reload())
|
||||
|
||||
@patch.object(Postgresql, 'is_running')
|
||||
def test_is_healthy(self, mock_is_running):
|
||||
mock_is_running.return_value = True
|
||||
def test_is_healthy(self):
|
||||
self.assertTrue(self.p.is_healthy())
|
||||
mock_is_running.return_value = False
|
||||
self.p.is_running = false
|
||||
self.assertFalse(self.p.is_healthy())
|
||||
|
||||
def test_promote(self):
|
||||
@@ -353,23 +338,13 @@ class TestPostgresql(unittest.TestCase):
|
||||
def test_last_operation(self):
|
||||
self.assertEquals(self.p.last_operation(), '0')
|
||||
|
||||
@patch('os.path.isfile', Mock(return_value=True))
|
||||
@patch('os.kill', Mock(side_effect=Exception))
|
||||
@patch('os.getpid', Mock(return_value=2))
|
||||
@patch('os.getppid', Mock(return_value=2))
|
||||
@patch.object(builtins, 'open', mock_open(read_data='-1'))
|
||||
@patch.object(Postgresql, '_version_file_exists', Mock(return_value=True))
|
||||
def test_is_running(self):
|
||||
self.assertFalse(self.p.is_running())
|
||||
|
||||
@patch('subprocess.Popen', Mock(side_effect=OSError))
|
||||
@patch('subprocess.Popen', Mock(side_effect=OSError()))
|
||||
def test_call_nowait(self):
|
||||
self.assertFalse(self.p.call_nowait('on_start'))
|
||||
|
||||
def test_non_existing_callback(self):
|
||||
self.assertFalse(self.p.call_nowait('foobar'))
|
||||
|
||||
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
|
||||
def test_is_leader_exception(self):
|
||||
self.p.start()
|
||||
self.p.query = Mock(side_effect=psycopg2.OperationalError("not supported"))
|
||||
@@ -381,53 +356,46 @@ class TestPostgresql(unittest.TestCase):
|
||||
@patch('os.rename', Mock())
|
||||
@patch('os.path.isdir', Mock(return_value=True))
|
||||
def test_move_data_directory(self):
|
||||
self.p.is_running = false
|
||||
self.p.move_data_directory()
|
||||
with patch('os.rename', Mock(side_effect=OSError)):
|
||||
with patch('os.rename', Mock(side_effect=OSError())):
|
||||
self.p.move_data_directory()
|
||||
|
||||
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
|
||||
@patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
|
||||
def test_bootstrap(self):
|
||||
with patch('subprocess.call', Mock(return_value=1)):
|
||||
self.assertRaises(PostgresException, self.p.bootstrap, {})
|
||||
|
||||
self.p.bootstrap({'users': {'replicator': {'password': 'rep-pass', 'options': ['replication']}},
|
||||
'pg_hba': ['host replication replicator 127.0.0.1/32 md5',
|
||||
'hostssl all all 0.0.0.0/0 md5',
|
||||
'host all all 0.0.0.0/0 md5']})
|
||||
with open(os.path.join(self.data_dir, 'pg_hba.conf')) as f:
|
||||
lines = f.readlines()
|
||||
assert 'host replication replicator 127.0.0.1/32 md5\n' in lines
|
||||
assert 'host all all 0.0.0.0/0 md5\n' in lines
|
||||
|
||||
@patch('patroni.postgresql.Postgresql.create_replica', Mock(return_value=0))
|
||||
def test_clone(self):
|
||||
self.p.clone(self.leader)
|
||||
|
||||
@patch('os.listdir', Mock(return_value=['recovery.conf']))
|
||||
@patch('os.path.exists', Mock(return_value=True))
|
||||
def test_get_postgres_role_from_data_directory(self):
|
||||
self.assertEquals(self.p.get_postgres_role_from_data_directory(), 'replica')
|
||||
self.assertRaises(PostgresException, self.p.bootstrap)
|
||||
self.p.bootstrap()
|
||||
self.p.bootstrap(self.leader)
|
||||
|
||||
def test_remove_data_directory(self):
|
||||
self.p.data_dir = 'data_dir'
|
||||
self.p.remove_data_directory()
|
||||
open(self.data_dir, 'w').close()
|
||||
os.mkdir(self.p.data_dir)
|
||||
self.p.remove_data_directory()
|
||||
os.symlink('unexisting', self.data_dir)
|
||||
with patch('os.unlink', Mock(side_effect=OSError)):
|
||||
open(self.p.data_dir, 'w').close()
|
||||
self.p.remove_data_directory()
|
||||
os.symlink('unexisting', self.p.data_dir)
|
||||
with patch('os.unlink', Mock(side_effect=Exception)):
|
||||
self.p.remove_data_directory()
|
||||
self.p.remove_data_directory()
|
||||
|
||||
@patch('patroni.postgresql.Postgresql._version_file_exists', Mock(return_value=True))
|
||||
def test_controldata(self):
|
||||
with patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string)):
|
||||
data = self.p.controldata()
|
||||
self.assertEquals(len(data), 50)
|
||||
self.assertEquals(data['Database cluster state'], 'shut down in recovery')
|
||||
self.assertEquals(data['wal_log_hints setting'], 'on')
|
||||
self.assertEquals(int(data['Database block size']), 8192)
|
||||
@patch('subprocess.check_output', MagicMock(return_value=0, side_effect=pg_controldata_string))
|
||||
@patch('subprocess.check_output', side_effect=subprocess.CalledProcessError)
|
||||
@patch('subprocess.check_output', side_effect=Exception('Failed'))
|
||||
def test_controldata(self, check_output_call_error, check_output_generic_exception):
|
||||
data = self.p.controldata()
|
||||
self.assertEquals(len(data), 50)
|
||||
self.assertEquals(data['Database cluster state'], 'shut down in recovery')
|
||||
self.assertEquals(data['wal_log_hints setting'], 'on')
|
||||
self.assertEquals(int(data['Database block size']), 8192)
|
||||
|
||||
with patch('subprocess.check_output', Mock(side_effect=subprocess.CalledProcessError(1, ''))):
|
||||
self.assertEquals(self.p.controldata(), {})
|
||||
subprocess.check_output = check_output_call_error
|
||||
data = self.p.controldata()
|
||||
self.assertEquals(data, dict())
|
||||
|
||||
subprocess.check_output = check_output_generic_exception
|
||||
self.assertRaises(Exception, self.p.controldata())
|
||||
|
||||
def test_read_postmaster_opts(self):
|
||||
m = mock_open(read_data=postmaster_opts_string())
|
||||
@@ -437,38 +405,47 @@ class TestPostgresql(unittest.TestCase):
|
||||
self.assertEquals(int(data['max_replication_slots']), 5)
|
||||
self.assertEqual(data.get('D'), None)
|
||||
|
||||
m.side_effect = IOError
|
||||
m.side_effect = IOError("foo")
|
||||
data = self.p.read_postmaster_opts()
|
||||
self.assertEqual(data, dict())
|
||||
|
||||
m.side_effect = Exception("foo")
|
||||
self.assertRaises(Exception, self.p.read_postmaster_opts())
|
||||
|
||||
@patch('subprocess.Popen')
|
||||
@patch.object(builtins, 'open', MagicMock(return_value=42))
|
||||
def test_single_user_mode(self, subprocess_popen_mock):
|
||||
subprocess_popen_mock.return_value.wait.return_value = 0
|
||||
self.assertEquals(self.p.single_user_mode(options=dict(archive_mode='on', archive_command='false')), 0)
|
||||
subprocess_popen_mock.assert_called_once_with(['postgres', '--single', '-D', self.data_dir,
|
||||
subprocess_popen_mock.assert_called_once_with(['postgres', '--single', '-D', self.p.data_dir,
|
||||
'-c', 'archive_command=false', '-c', 'archive_mode=on',
|
||||
'postgres'], stdin=subprocess.PIPE,
|
||||
stdout=42,
|
||||
stderr=subprocess.STDOUT)
|
||||
subprocess_popen_mock.reset_mock()
|
||||
self.assertEquals(self.p.single_user_mode(command="CHECKPOINT"), 0)
|
||||
subprocess_popen_mock.assert_called_once_with(['postgres', '--single', '-D', self.data_dir,
|
||||
subprocess_popen_mock.assert_called_once_with(['postgres', '--single', '-D', self.p.data_dir,
|
||||
'postgres'], stdin=subprocess.PIPE,
|
||||
stdout=42,
|
||||
stderr=subprocess.STDOUT)
|
||||
subprocess_popen_mock.return_value = None
|
||||
self.assertEquals(self.p.single_user_mode(), 1)
|
||||
|
||||
def fake_listdir(path):
|
||||
if path.endswith(os.path.join('pg_xlog', 'archive_status')):
|
||||
return ["a", "b", "c"]
|
||||
return []
|
||||
|
||||
@patch('os.listdir', MagicMock(side_effect=fake_listdir))
|
||||
@patch('os.path.isdir', MagicMock(return_value=True))
|
||||
@patch('os.unlink', return_value=True)
|
||||
@patch('os.remove', return_value=True)
|
||||
@patch('os.path.islink', return_value=False)
|
||||
@patch('os.path.isfile', return_value=True)
|
||||
def test_cleanup_archive_status(self, mock_file, mock_link, mock_remove, mock_unlink):
|
||||
ap = os.path.join(self.data_dir, 'pg_xlog', 'archive_status/')
|
||||
ap = os.path.join(self.p.data_dir, 'pg_xlog', 'archive_status/')
|
||||
self.p.cleanup_archive_status()
|
||||
mock_remove.assert_has_calls([mock.call(ap + 'a'), mock.call(ap + 'b'), mock.call(ap + 'c')])
|
||||
mock_remove.assert_has_calls([mock.call(ap+'a'), mock.call(ap+'b'), mock.call(ap+'c')])
|
||||
mock_unlink.assert_not_called()
|
||||
|
||||
mock_remove.reset_mock()
|
||||
@@ -476,64 +453,30 @@ class TestPostgresql(unittest.TestCase):
|
||||
mock_file.return_value = False
|
||||
mock_link.return_value = True
|
||||
self.p.cleanup_archive_status()
|
||||
mock_unlink.assert_has_calls([mock.call(ap + 'a'), mock.call(ap + 'b'), mock.call(ap + 'c')])
|
||||
mock_unlink.assert_has_calls([mock.call(ap+'a'), mock.call(ap+'b'), mock.call(ap+'c')])
|
||||
mock_remove.assert_not_called()
|
||||
|
||||
mock_unlink.reset_mock()
|
||||
mock_remove.reset_mock()
|
||||
|
||||
mock_file.side_effect = OSError
|
||||
mock_link.side_effect = OSError
|
||||
mock_file.side_effect = Exception("foo")
|
||||
mock_link.side_effect = Exception("foo")
|
||||
self.p.cleanup_archive_status()
|
||||
mock_unlink.assert_not_called()
|
||||
mock_remove.assert_not_called()
|
||||
|
||||
@patch('patroni.postgresql.Postgresql._version_file_exists', Mock(return_value=True))
|
||||
@patch('subprocess.check_output', MagicMock(return_value=0, side_effect=pg_controldata_string))
|
||||
def test_sysid(self):
|
||||
self.assertEqual(self.p.sysid, "6200971513092291716")
|
||||
|
||||
@patch('os.path.isfile', Mock(return_value=True))
|
||||
@patch('shutil.copy', Mock(side_effect=IOError))
|
||||
def test_save_configuration_files(self):
|
||||
@patch('os.path.isfile', MagicMock(return_value=True))
|
||||
@patch('shutil.copy', side_effect=Exception)
|
||||
def test_save_configuration_files(self, mock_copy):
|
||||
shutil.copy = mock_copy
|
||||
self.p.save_configuration_files()
|
||||
|
||||
@patch('os.path.isfile', Mock(side_effect=[False, True]))
|
||||
@patch('shutil.copy', Mock(side_effect=IOError))
|
||||
def test_restore_configuration_files(self):
|
||||
@patch('os.path.isfile', MagicMock(side_effect=is_file_raise_on_backup))
|
||||
@patch('shutil.copy', side_effect=Exception)
|
||||
def test_restore_configuration_files(self, mock_copy):
|
||||
shutil.copy = mock_copy
|
||||
self.p.restore_configuration_files()
|
||||
|
||||
def test_can_create_replica_without_replication_connection(self):
|
||||
self.p.config['create_replica_method'] = []
|
||||
self.assertFalse(self.p.can_create_replica_without_replication_connection())
|
||||
self.p.config['create_replica_method'] = ['wale', 'basebackup']
|
||||
self.p.config['wale'] = {'command': 'foo', 'no_master': 1}
|
||||
self.assertTrue(self.p.can_create_replica_without_replication_connection())
|
||||
|
||||
def test_replica_method_can_work_without_replication_connection(self):
|
||||
self.assertFalse(self.p.replica_method_can_work_without_replication_connection('basebackup'))
|
||||
self.assertFalse(self.p.replica_method_can_work_without_replication_connection('foobar'))
|
||||
self.p.config['foo'] = {'command': 'bar', 'no_master': 1}
|
||||
self.assertTrue(self.p.replica_method_can_work_without_replication_connection('foo'))
|
||||
self.p.config['foo'] = {'command': 'bar'}
|
||||
self.assertFalse(self.p.replica_method_can_work_without_replication_connection('foo'))
|
||||
|
||||
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
|
||||
def test_reload_config(self):
|
||||
parameters = self._PARAMETERS.copy()
|
||||
parameters.pop('f.oo')
|
||||
self.p.reload_config({'retry_timeout': 10, 'listen': '*', 'parameters': parameters})
|
||||
parameters['b.ar'] = 'bar'
|
||||
self.p.reload_config({'retry_timeout': 10, 'listen': '*', 'parameters': parameters})
|
||||
parameters['autovacuum'] = 'on'
|
||||
self.p.reload_config({'retry_timeout': 10, 'listen': '*', 'parameters': parameters})
|
||||
parameters['autovacuum'] = 'off'
|
||||
parameters.pop('search_path')
|
||||
self.p.reload_config({'retry_timeout': 10, 'listen': '*:5433', 'parameters': parameters})
|
||||
|
||||
@patch.object(Postgresql, '_version_file_exists', Mock(return_value=True))
|
||||
def test_get_major_version(self):
|
||||
with patch.object(builtins, 'open', mock_open(read_data='9.4')):
|
||||
self.assertEquals(self.p.get_major_version(), 9.4)
|
||||
with patch.object(builtins, 'open', Mock(side_effect=Exception)):
|
||||
self.assertEquals(self.p.get_major_version(), 0.0)
|
||||
|
||||
+17
-12
@@ -2,7 +2,7 @@ import unittest
|
||||
|
||||
from mock import Mock, patch
|
||||
from patroni.exceptions import PatroniException
|
||||
from patroni.utils import reap_children, Retry, RetryFailedError, sigchld_handler, sleep
|
||||
from patroni.utils import Retry, RetryFailedError, reap_children, sigchld_handler, sigterm_handler, sleep
|
||||
|
||||
|
||||
def time_sleep(_):
|
||||
@@ -11,23 +11,25 @@ def time_sleep(_):
|
||||
|
||||
class TestUtils(unittest.TestCase):
|
||||
|
||||
def test_sigterm_handler(self):
|
||||
self.assertRaises(SystemExit, sigterm_handler, None, None)
|
||||
|
||||
@patch('time.sleep', Mock())
|
||||
def test_reap_children(self):
|
||||
self.assertIsNone(reap_children())
|
||||
reap_children()
|
||||
with patch('os.waitpid', Mock(return_value=(0, 0))):
|
||||
sigchld_handler(None, None)
|
||||
self.assertIsNone(reap_children())
|
||||
reap_children()
|
||||
|
||||
@patch('time.sleep', time_sleep)
|
||||
def test_sleep(self):
|
||||
self.assertIsNone(sleep(0.01))
|
||||
sleep(0.01)
|
||||
|
||||
|
||||
@patch('time.sleep', Mock())
|
||||
class TestRetrySleeper(unittest.TestCase):
|
||||
|
||||
@staticmethod
|
||||
def _fail(times=1):
|
||||
def _fail(self, times=1):
|
||||
scope = dict(times=0)
|
||||
|
||||
def inner():
|
||||
@@ -38,33 +40,36 @@ class TestRetrySleeper(unittest.TestCase):
|
||||
raise PatroniException('Failed!')
|
||||
return inner
|
||||
|
||||
def _makeOne(self, *args, **kwargs):
|
||||
return Retry(*args, **kwargs)
|
||||
|
||||
def test_reset(self):
|
||||
retry = Retry(delay=0, max_tries=2)
|
||||
retry = self._makeOne(delay=0, max_tries=2)
|
||||
retry(self._fail())
|
||||
self.assertEquals(retry._attempts, 1)
|
||||
retry.reset()
|
||||
self.assertEquals(retry._attempts, 0)
|
||||
|
||||
def test_too_many_tries(self):
|
||||
retry = Retry(delay=0)
|
||||
retry = self._makeOne(delay=0)
|
||||
self.assertRaises(RetryFailedError, retry, self._fail(times=999))
|
||||
self.assertEquals(retry._attempts, 1)
|
||||
|
||||
def test_maximum_delay(self):
|
||||
retry = Retry(delay=10, max_tries=100)
|
||||
retry = self._makeOne(delay=10, max_tries=100)
|
||||
retry(self._fail(times=10))
|
||||
self.assertTrue(retry._cur_delay < 4000, retry._cur_delay)
|
||||
# gevent's sleep function is picky about the type
|
||||
self.assertEquals(type(retry._cur_delay), float)
|
||||
|
||||
def test_deadline(self):
|
||||
retry = Retry(deadline=0.0001)
|
||||
retry = self._makeOne(deadline=0.0001)
|
||||
self.assertRaises(RetryFailedError, retry, self._fail(times=100))
|
||||
|
||||
def test_copy(self):
|
||||
def _sleep(t):
|
||||
pass
|
||||
None
|
||||
|
||||
retry = Retry(sleep_func=_sleep)
|
||||
retry = self._makeOne(sleep_func=_sleep)
|
||||
rcopy = retry.copy()
|
||||
self.assertTrue(rcopy.sleep_func is _sleep)
|
||||
|
||||
+25
-14
@@ -1,9 +1,25 @@
|
||||
import unittest
|
||||
from mock import MagicMock, patch, PropertyMock
|
||||
import os
|
||||
import psycopg2
|
||||
import subprocess
|
||||
import unittest
|
||||
from patroni.scripts.wale_restore import WALERestore
|
||||
|
||||
from mock import MagicMock, patch, PropertyMock
|
||||
from patroni.scripts.wale_restore import WALERestore, main as _main
|
||||
|
||||
def fake_cursor_fetchone(*args, **kwargs):
|
||||
return ('16777216',)
|
||||
|
||||
|
||||
def fake_call_fail_for_wal_e(*args, **kwargs):
|
||||
if len(args) > 0 and 'backup-fetch' in args[0]:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def fake_call_fail_for_base_backup(*args, **kwargs):
|
||||
if len(args) > 0 and 'backup-fetch' in args[0]:
|
||||
return 0
|
||||
return 1
|
||||
|
||||
|
||||
def fake_backup_data(self, *args, **kwargs):
|
||||
@@ -12,19 +28,16 @@ def fake_backup_data(self, *args, **kwargs):
|
||||
base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 00000001000000000000007F 00000040 00000001000000000000007F 00000240
|
||||
"""
|
||||
|
||||
|
||||
def fake_backup_data_2(self, *args, **kwargs):
|
||||
""" return the fake result of WAL-E backup-list"""
|
||||
return """name last_modified expanded_size_bytes wal_segment_backup_start wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop """
|
||||
|
||||
|
||||
def fake_backup_data_3(self, *args, **kwargs):
|
||||
""" return the fake result of WAL-E backup-list"""
|
||||
return """name last_modified expanded_size_bytes wal_segment_backup_start wal_segment_offset_backup_start wal_segment_backup_stop
|
||||
base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 00000001000000000000007F 00000040 00000001000000000000007F 00000240
|
||||
"""
|
||||
|
||||
|
||||
def fake_backup_data_4(self, *args, **kwargs):
|
||||
""" return the fake result of WAL-E backup-list"""
|
||||
return """name last_modified expanded_size_foo wal_segment_backup_start wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop
|
||||
@@ -36,6 +49,7 @@ base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 000
|
||||
@patch('os.makedirs', MagicMock(return_value=True))
|
||||
@patch('os.path.exists', MagicMock(return_value=True))
|
||||
@patch('os.path.isdir', MagicMock(return_value=True))
|
||||
@patch('psycopg2.extensions.cursor.fetchone', MagicMock(side_effect=fake_cursor_fetchone))
|
||||
@patch('psycopg2.extensions.cursor', MagicMock(autospec=True))
|
||||
@patch('psycopg2.extensions.connection', MagicMock(autospec=True))
|
||||
@patch('psycopg2.connect', MagicMock(autospec=True))
|
||||
@@ -43,7 +57,11 @@ base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 000
|
||||
class TestWALERestore(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.wale_restore = WALERestore("batman", "/data", "host=batman port=5432 user=batman", "/etc", 100, 100, 1, 0)
|
||||
self.wale_restore = WALERestore("batman", "/data",
|
||||
"host=batman port=5432 user=batman", "/etc", 100, 100, 1)
|
||||
|
||||
def tearDown(self):
|
||||
pass
|
||||
|
||||
def test_should_use_s3_to_create_replica(self):
|
||||
with patch('psycopg2.connect', MagicMock(side_effect=psycopg2.Error("foo"))):
|
||||
@@ -58,8 +76,6 @@ class TestWALERestore(unittest.TestCase):
|
||||
self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
|
||||
|
||||
self.wale_restore.should_use_s3_to_create_replica()
|
||||
self.wale_restore.no_master = 1
|
||||
self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
|
||||
|
||||
def test_create_replica_with_s3(self):
|
||||
with patch('subprocess.call', MagicMock(return_value=0)):
|
||||
@@ -73,8 +89,3 @@ class TestWALERestore(unittest.TestCase):
|
||||
with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', MagicMock(return_value=True)):
|
||||
with patch.object(self.wale_restore, 'create_replica_with_s3', MagicMock(return_value=0)):
|
||||
self.assertEqual(self.wale_restore.run(), 0)
|
||||
|
||||
@patch('sys.exit', MagicMock())
|
||||
@patch.object(WALERestore, 'run', MagicMock(return_value=0))
|
||||
def test_main(self):
|
||||
self.assertEqual(_main(), None)
|
||||
|
||||
+29
-48
@@ -1,28 +1,26 @@
|
||||
import six
|
||||
import unittest
|
||||
|
||||
from mock import Mock, patch
|
||||
from patroni.dcs import Leader
|
||||
from patroni.zookeeper import ExhibitorEnsembleProvider, ZooKeeper, ZooKeeperError
|
||||
from kazoo.client import KazooState
|
||||
from kazoo.exceptions import NoNodeError, NodeExistsError
|
||||
from kazoo.handlers.threading import SequentialThreadingHandler
|
||||
from kazoo.protocol.states import ZnodeStat
|
||||
from mock import Mock, patch
|
||||
from patroni.dcs.zookeeper import Leader, PatroniSequentialThreadingHandler, ZooKeeper, ZooKeeperError
|
||||
from test_etcd import SleepException, requests_get
|
||||
|
||||
|
||||
class MockKazooClient(Mock):
|
||||
|
||||
leader = False
|
||||
exists = True
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(MockKazooClient, self).__init__()
|
||||
handler = Mock()
|
||||
|
||||
@property
|
||||
def client_id(self):
|
||||
return (-1, '')
|
||||
|
||||
@staticmethod
|
||||
def retry(func, *args, **kwargs):
|
||||
def retry(self, func, *args, **kwargs):
|
||||
func(*args, **kwargs)
|
||||
|
||||
def get(self, path, watch=None):
|
||||
@@ -35,6 +33,8 @@ class MockKazooClient(Mock):
|
||||
b'postgres://repuser:rep-pass@localhost:5434/postgres?application_name=http://127.0.0.1:8009/patroni',
|
||||
ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0 if self.exists else -1, 0, 0, 0)
|
||||
)
|
||||
elif path.endswith('/optime/leader'):
|
||||
return (b'1', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))
|
||||
elif path.endswith('/leader'):
|
||||
if self.leader:
|
||||
return (b'foo', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0))
|
||||
@@ -43,8 +43,7 @@ class MockKazooClient(Mock):
|
||||
return (b'foo', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))
|
||||
return (b'', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))
|
||||
|
||||
@staticmethod
|
||||
def get_children(path, watch=None, include_data=False):
|
||||
def get_children(self, path, watch=None, include_data=False):
|
||||
if not isinstance(path, six.string_types):
|
||||
raise TypeError("Invalid type for 'path' (string expected)")
|
||||
if path.startswith('/no_node'):
|
||||
@@ -63,17 +62,17 @@ class MockKazooClient(Mock):
|
||||
elif value == b'retry' or (value == b'exists' and self.exists):
|
||||
raise NodeExistsError
|
||||
|
||||
@staticmethod
|
||||
def set(path, value, version=-1):
|
||||
def set(self, path, value, version=-1):
|
||||
if not isinstance(path, six.string_types):
|
||||
raise TypeError("Invalid type for 'path' (string expected)")
|
||||
if not isinstance(value, (six.binary_type,)):
|
||||
raise TypeError("Invalid type for 'value' (must be a byte string)")
|
||||
if path == '/service/bla/optime/leader':
|
||||
raise Exception
|
||||
if path == '/service/test/members/bar' and value == b'retry':
|
||||
return
|
||||
if path in ('/service/test/failover', '/service/test/config'):
|
||||
if path == '/service/test/members/bar':
|
||||
if value == b'retry':
|
||||
return
|
||||
if path == '/service/test/failover':
|
||||
if value == b'Exception':
|
||||
raise Exception
|
||||
elif value == b'ok':
|
||||
@@ -85,41 +84,34 @@ class MockKazooClient(Mock):
|
||||
raise TypeError("Invalid type for 'path' (string expected)")
|
||||
self.exists = False
|
||||
if path == '/service/test/leader':
|
||||
if self.leader:
|
||||
return
|
||||
self.leader = True
|
||||
raise Exception
|
||||
elif path == '/service/test/members/buzz':
|
||||
raise Exception
|
||||
elif path.endswith('/') or path.endswith('/initialize') or path == '/service/test/members/bar':
|
||||
elif path.endswith('/initialize') or path == '/service/test/members/bar':
|
||||
raise NoNodeError
|
||||
|
||||
|
||||
class TestPatroniSequentialThreadingHandler(unittest.TestCase):
|
||||
@patch('requests.get', requests_get)
|
||||
@patch('patroni.zookeeper.sleep', Mock(side_effect=SleepException()))
|
||||
class TestExhibitorEnsembleProvider(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.handler = PatroniSequentialThreadingHandler(10)
|
||||
|
||||
@patch.object(SequentialThreadingHandler, 'create_connection', Mock())
|
||||
def test_create_connection(self):
|
||||
self.assertIsNotNone(self.handler.create_connection(()))
|
||||
self.assertIsNotNone(self.handler.create_connection((), 40))
|
||||
def test_init(self):
|
||||
self.assertRaises(SleepException, ExhibitorEnsembleProvider, ['localhost'], 8181)
|
||||
|
||||
|
||||
class TestZooKeeper(unittest.TestCase):
|
||||
|
||||
@patch('patroni.dcs.zookeeper.KazooClient', MockKazooClient)
|
||||
@patch('requests.get', requests_get)
|
||||
@patch('patroni.zookeeper.KazooClient', MockKazooClient)
|
||||
def setUp(self):
|
||||
self.zk = ZooKeeper({'hosts': ['localhost:2181'], 'scope': 'test',
|
||||
'name': 'foo', 'ttl': 30, 'retry_timeout': 10})
|
||||
self.zk = ZooKeeper('foo', {'exhibitor': {'hosts': ['localhost', 'exhibitor'], 'port': 8181}, 'scope': 'test'})
|
||||
|
||||
def test_session_listener(self):
|
||||
self.zk.session_listener(KazooState.SUSPENDED)
|
||||
|
||||
def test_set_ttl(self):
|
||||
self.zk.set_ttl(20)
|
||||
|
||||
def test_set_retry_timeout(self):
|
||||
self.zk.set_retry_timeout(10)
|
||||
|
||||
def test_get_node(self):
|
||||
self.assertIsNone(self.zk.get_node('/no_node'))
|
||||
|
||||
@@ -134,23 +126,17 @@ class TestZooKeeper(unittest.TestCase):
|
||||
|
||||
def test_get_cluster(self):
|
||||
self.assertRaises(ZooKeeperError, self.zk.get_cluster)
|
||||
self.zk.exhibitor.poll = lambda: True
|
||||
cluster = self.zk.get_cluster()
|
||||
self.assertIsInstance(cluster.leader, Leader)
|
||||
self.zk.touch_member('foo')
|
||||
|
||||
def test_delete_leader(self):
|
||||
self.assertTrue(self.zk.delete_leader())
|
||||
self.zk.delete_leader()
|
||||
|
||||
def test_set_failover_value(self):
|
||||
self.zk.set_failover_value('')
|
||||
self.zk.set_failover_value('ok')
|
||||
self.zk.set_failover_value('Exception')
|
||||
|
||||
def test_set_config_value(self):
|
||||
self.zk.set_config_value('')
|
||||
self.zk.set_config_value('ok')
|
||||
self.zk.set_config_value('Exception')
|
||||
|
||||
def test_initialize(self):
|
||||
self.assertFalse(self.zk.initialize())
|
||||
|
||||
@@ -164,18 +150,16 @@ class TestZooKeeper(unittest.TestCase):
|
||||
self.zk._name = 'bar'
|
||||
self.zk.touch_member('new')
|
||||
self.zk._name = 'na'
|
||||
self.zk._client.exists = 1
|
||||
self.zk.client.exists = 1
|
||||
self.zk.touch_member('exists')
|
||||
self.zk._name = 'bar'
|
||||
self.zk.touch_member('retry')
|
||||
self.zk._fetch_cluster = True
|
||||
self.zk.fetch_cluster = True
|
||||
self.zk.get_cluster()
|
||||
self.zk.touch_member('retry')
|
||||
|
||||
def test_take_leader(self):
|
||||
self.zk.take_leader()
|
||||
with patch.object(MockKazooClient, 'create', Mock(side_effect=Exception)):
|
||||
self.zk.take_leader()
|
||||
|
||||
def test_update_leader(self):
|
||||
self.assertTrue(self.zk.update_leader())
|
||||
@@ -186,9 +170,6 @@ class TestZooKeeper(unittest.TestCase):
|
||||
self.zk._base_path = self.zk._base_path.replace('test', 'bla')
|
||||
self.zk.write_leader_optime('2')
|
||||
|
||||
def test_delete_cluster(self):
|
||||
self.assertTrue(self.zk.delete_cluster())
|
||||
|
||||
def test_watch(self):
|
||||
self.zk.watch(0)
|
||||
self.zk.event.isSet = lambda: True
|
||||
|
||||
Reference in New Issue
Block a user