Compare commits

..
9 Commits
85 changed files with 1822 additions and 6976 deletions
-3
View File
@@ -51,6 +51,3 @@ scm-source.json
docs/build/
docs/source/_static/
docs/source/_templates/
# Pycharm IDE
.idea/
+17 -52
View File
@@ -1,28 +1,17 @@
sudo: true
sudo: false
dist: trusty
language: python
python:
- "3.4" # 2.7 and 3.5 are preinstalled by default
env:
global:
- ETCDVERSION=3.0.17 ZKVERSION=3.4.11 CONSULVERSION=0.7.4
- PYVERSIONS="2.7 3.5 3.6"
- EXCLUDE_BEHAVE="3.5"
- BOTO_CONFIG=/doesnotexist
matrix:
include:
- python: "3.5"
env: TEST_SUITE="python setup.py"
- python: "3.6"
env: DCS="etcd" TEST_SUITE="behave"
- python: "3.6"
env: DCS="exhibitor" TEST_SUITE="behave"
- python: "3.6"
env: DCS="consul" TEST_SUITE="behave"
- python: "3.6"
env: DCS="kubernetes" TEST_SUITE="behave"
branches:
only:
- master
- /^v\d+\.\d+(\.\d+)?$/
- ETCDVERSION=3.0.17 ZKVERSION=3.4.9 CONSULVERSION=0.7.4
- PYVERSIONS="2.7 3.4 3.5"
matrix:
- TEST_SUITE="python setup.py"
- DCS="etcd" TEST_SUITE="behave"
- DCS="exhibitor" TEST_SUITE="behave"
- DCS="consul" TEST_SUITE="behave"
cache:
directories:
- $HOME/mycache
@@ -30,7 +19,7 @@ before_cache:
- |
rm -fr $HOME/mycache/python*
for pv in $PYVERSIONS; do
if [[ $TEST_SUITE != "behave" || $pv != $EXCLUDE_BEHAVE ]]; then
if [[ $TEST_SUITE != "behave" || $pv != "3.4" ]]; then
fpv=$(basename $(readlink $HOME/virtualenv/python${pv}))
mv $HOME/virtualenv/${fpv} $HOME/mycache/${fpv}
fi
@@ -62,26 +51,6 @@ install:
ln -s $EC etcd
}
function get_kubernetes() {
wget -O localkube "https://storage.googleapis.com/minikube/k8sReleases/v1.7.0/localkube-linux-amd64"
chmod +x localkube
sudo nohup ./localkube --logtostderr=true --enable-dns=false > localkube.log 2>&1 &
echo "Waiting for localkube to start..."
if ! timeout 120 sh -c "while ! curl -ks http://127.0.0.1:8080/ >/dev/null; do sleep 1; done"; then
sudo cat localkube.log
echo "localkube did not start"
exit 1
fi
echo "Check certificate permissions"
sudo chmod 644 /var/lib/localkube/certs/*
sudo ls -altr /var/lib/localkube/certs/
echo "Set up .kube/config"
mkdir ~/.kube
echo -e "apiVersion: v1\nclusters:\n- cluster:\n certificate-authority: /var/lib/localkube/certs/ca.crt\n server: https://127.0.0.1:8443\n name: local\ncontexts:\n- context:\n cluster: local\n user: myself\n name: local\ncurrent-context: local\nkind: Config\npreferences: {}\nusers:\n- name: myself\n user:\n client-certificate: /var/lib/localkube/certs/apiserver.crt\n client-key: /var/lib/localkube/certs/apiserver.key\n" > ~/.kube/config
}
function get_exhibitor() {
ZC=~/mycache/zookeeper-${ZKVERSION}
if [[ ! -d $ZC ]]; then
@@ -96,6 +65,7 @@ install:
echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \
| nc -l 8181 &> /dev/null
done&
ZK_PID=$!
}
attempt_num=1
@@ -107,7 +77,7 @@ install:
fi
for pv in $PYVERSIONS; do
if [[ $TEST_SUITE != "behave" || $pv != $EXCLUDE_BEHAVE ]]; then
if [[ $TEST_SUITE != "behave" || $pv != "3.4" ]]; then
fpv=$(basename $(readlink $HOME/virtualenv/python$pv))
if [[ -d ~/mycache/${fpv} ]]; then
mv ~/virtualenv/${fpv} ~/virtualenv/${fpv}.bckp
@@ -123,21 +93,17 @@ install:
script:
- |
for pv in $PYVERSIONS; do
if [[ $TEST_SUITE == "behave" && $pv == $EXCLUDE_BEHAVE ]]; then
continue
fi
source ~/virtualenv/python${pv}/bin/activate
if [[ $TEST_SUITE != "behave" ]]; then
echo Running unit tests using python${pv}
$TEST_SUITE test
$TEST_SUITE flake8
elif [[ $pv != $EXCLUDE_BEHAVE ]]; then
elif [[ $pv != "3.4" ]]; then
echo Running acceptance tests using python${pv}
if ! PATH=.:/usr/lib/postgresql/9.6/bin:$PATH $TEST_SUITE; then
# output all log files when tests are failing
grep . features/output/*_failed/*postgres?.*
grep . features/output/*/*postgres?.*
exit 1
fi
fi
@@ -146,8 +112,7 @@ script:
set +e
after_success:
# before_cache is executed earlier than after_success, so we need to restore one of virtualenv directories
- fpv=$(basename $(readlink $HOME/virtualenv/python3.6)) && mv $HOME/mycache/${fpv} $HOME/virtualenv/${fpv}
- fpv=$(basename $(readlink $HOME/virtualenv/python3.5)) && mv $HOME/mycache/${fpv} $HOME/virtualenv/${fpv}
- coveralls
- if [[ $TEST_SUITE != "behave" ]]; then python-codacy-coverage -r coverage.xml; fi
- if [[ $DCS == "exhibitor" ]]; then ~/mycache/zookeeper-${ZKVERSION}/bin/zkServer.sh stop; fi
- sudo kill $(jobs -p)
- if [[ $DCS == "exhibitor" ]]; then ~/mycache/zookeeper-${ZKVERSION}/bin/zkServer.sh stop; kill -9 $ZK_PID; fi
+13
View File
@@ -0,0 +1,13 @@
# for github.com
approvals:
groups:
zalando:
minimum: 2
from:
orgs:
- "zalando"
# team should be valid team id in team service https://teams.auth.zalando.com/api/teams/:id
X-Zalando-Team: "acid"
# type should be one of [code, doc, config, tools, secrets]
# code will be the default value, if X-Zalando-Type is not found in .zappr.yml
X-Zalando-Type: code
+12 -9
View File
@@ -1,42 +1,45 @@
## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
## It has all the necessary components to play/debug with a single node appliance, running etcd
FROM postgres:10
FROM postgres:9.6
MAINTAINER Alexander Kukushkin <[email protected]>
RUN export DEBIAN_FRONTEND=noninteractive \
&& echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
&& apt-get update -y \
&& apt-get upgrade -y \
# postgres:10 is based on debian, which has patroni package. We will install all required dependencies
&& apt-get install -s patroni | sed -n -e '/^Inst patroni /d' -e 's/^Inst \([^ ]\+\) .*$/\1/p' \
| xargs apt-get install -y curl jq haproxy locales python3-etcd python3-kazoo \
&& apt-get install -y curl jq haproxy python-psycopg2 python-yaml python-requests python-six python-pysocks \
python-dateutil python-pip python-setuptools python-prettytable python-wheel python-psutil python locales \
## Make sure we have a en_US.UTF-8 locale available
&& localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
&& pip install 'python-etcd>=0.4.3,<0.5' click tzlocal cdiff \
&& mkdir -p /home/postgres \
&& chown postgres:postgres /home/postgres \
# Clean up
&& apt-get purge -y libpython2.7-stdlib libpython2.7-minimal \
&& apt-get remove -y python-pip python-setuptools \
&& apt-get autoremove -y \
&& apt-get clean -y \
&& rm -rf /var/lib/apt/lists/* /root/.cache
ENV ETCDVERSION 3.2.23
ENV ETCDVERSION 3.2.3
RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
| tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl
ENV CONFDVERSION 0.16.0
ENV CONFDVERSION 0.11.0
RUN curl -L https://github.com/kelseyhightower/confd/releases/download/v${CONFDVERSION}/confd-${CONFDVERSION}-linux-amd64 > /usr/local/bin/confd \
&& chmod +x /usr/local/bin/confd
ADD patronictl.py patroni.py docker/entrypoint.sh /
ADD patroni /patroni/
ADD extras/confd /etc/confd
RUN ln -s /patronictl.py /usr/local/bin/patronictl
RUN sed -i 's/env python/&3/' patroni*.py && ln -s /patronictl.py /usr/local/bin/patronictl && mkdir /data/ /run/haproxy \
&& touch /pgpass /patroni.yml && chown postgres:postgres -R /patroni/ /data/ /pgpass /patroni.yml /etc/haproxy /var/run/ /var/lib/ /var/log/
### Setting up a simple script that will serve as an entrypoint
RUN mkdir /data/ && touch /pgpass /patroni.yml \
&& chown postgres:postgres -R /patroni/ /data/ /pgpass /patroni.yml /etc/haproxy /var/run/ /var/lib/ /var/log/
EXPOSE 2379 5432 8008
+20 -56
View File
@@ -1,27 +1,27 @@
|Build Status| |Coverage Status|
Patroni: A Template for PostgreSQL HA with ZooKeeper, etcd or Consul
--------------------------------------------------------------------
------------------------------------------------------------
You can find a version of this documentation that is searchable and also easier to navigate at `patroni.readthedocs.io <https://patroni.readthedocs.io>`__.
There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__, `Consul <https://github.com/hashicorp/consul>`__ or `Kubernetes <https://kubernetes.io>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely.
**Note to Kubernetes users**: Patroni can run natively on top of Kubernetes. Take a look at the `Kubernetes <https://github.com/zalando/patroni/blob/master/docs/kubernetes.rst>`__ chapter of the Patroni documentation.
**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. To this end, there is a `Helm chart <https://github.com/kubernetes/charts/tree/master/incubator/patroni>`__ that uses Patroni and `Spilo <https://github.com/zalando/spilo/>`__ to provision a five-node PostgreSQL HA cluster in a Kubernetes+GCE environment. (The Helm chart deploys Spilo Docker images, not just "bare" Patroni.)
.. contents::
:local:
:depth: 1
:backlinks: none
=================
==============
How Patroni Works
=================
==============
Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features.
@@ -29,70 +29,34 @@ For an example of a Docker-based deployment with Patroni, see `Spilo <https://gi
For additional background info, see:
* `Elephants on Automatic: HA Clustered PostgreSQL with Helm <https://www.youtube.com/watch?v=CftcVhFMGSY>`_, talk by Josh Berkus and Oleksii Kliukin at KubeCon Berlin 2017
* `Elephants on Automatic: HA Clustered PostgreSQL with Helm <https://www.youtube.com/watch?v=CftcVhFMGSY>`_, talk by Josh Berkus and Oleksii Kliukin at KubeCon Berlin 2017
* `PostgreSQL HA with Kubernetes and Patroni <https://www.youtube.com/watch?v=iruaCgeG7qs>`__, talk by Josh Berkus at KubeCon 2016 (video)
* `Feb. 2016 Zalando Tech blog post <https://tech.zalando.de/blog/zalandos-patroni-a-template-for-high-availability-postgresql/>`__
==================
================
Development Status
==================
================
Patroni is in active development and accepts contributions. See our `Contributing <https://github.com/zalando/patroni/blob/master/docs/CONTRIBUTING.rst>`__ section below for more details.
We report new releases information `here <https://github.com/zalando/patroni/releases>`__.
=========
Community
=========
There are two places to connect with the Patroni community: `on github <https://github.com/zalando/patroni>`__, via Issues and PRs, and on channel #patroni in the `PostgreSQL Slack <https://postgres-slack.herokuapp.com/>`__. If you're using Patroni, or just interested, please join us.
===================================
===========================
Technical Requirements/Installation
===================================
===========================
**Pre-requirements for Mac OS**
**For Mac**
To install requirements on a Mac, run the following:
::
brew install postgresql etcd haproxy libyaml python
pip install psycopg2 pyyaml
**General installation for pip**
Patroni can be installed with pip:
::
pip install patroni[dependencies]
where dependencies can be either empty, or consist of one or more of the following:
etcd
`python-etcd` module in order to use Etcd as DCS
consul
`python-consul` module in order to use Consul as DCS
zookeeper
`kazoo` module in order to use Zookeeper as DCS
exhibitor
`kazoo` module in order to use Exhibitor as DCS (same dependencies as for Zookeeper)
kubernetes
`kubernetes` module in order to use Kubernetes as DCS in Patroni
aws
`boto` in order to use AWS callbacks
For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is:
::
pip install patroni[etcd,aws]
Note that external tools to call in the replica creation or custom bootstap scripts (i.e. WAL-E) should be installed independently of Patroni.
=======================
===================
Running and Configuring
=======================
===================
To get started, do the following from different terminals:
::
@@ -116,9 +80,9 @@ run:
> psql --host 127.0.0.1 --port 5000 postgres
==================
===============
YAML Configuration
==================
===============
Go `here <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
@@ -128,15 +92,15 @@ Environment Configuration
Go `here <https://github.com/zalando/patroni/blob/master/docs/ENVIRONMENT.rst>`__ for comprehensive information about configuring(overriding) settings via environment variables.
===================
===============
Replication Choices
===================
===============
Patroni uses Postgres' streaming replication, which is asynchronous by default. Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the leader. This setting should be increased or decreased based on business requirements. It's also possible to use synchronous replication for better durability guarantees. See `replication modes documentation <https://github.com/zalando/patroni/blob/master/docs/replication_modes.rst>`__ for details.
======================================
===============================
Applications Should Not Use Superusers
======================================
===============================
When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
+4 -4
View File
@@ -15,7 +15,7 @@ dbnode1:
- ./patroni:/patroni
env_file: docker/patroni-secrets.env
environment:
PATRONI_ETCD_URL: http://patroni_etcd:2379
PATRONI_ETCD_HOST: patroni_etcd:2379
PATRONI_NAME: dbnode1
PATRONI_SCOPE: testcluster
@@ -28,7 +28,7 @@ dbnode2:
- ./patroni:/patroni
env_file: docker/patroni-secrets.env
environment:
PATRONI_ETCD_URL: http://patroni_etcd:2379
PATRONI_ETCD_HOST: patroni_etcd:2379
PATRONI_NAME: dbnode2
PATRONI_SCOPE: testcluster
@@ -41,7 +41,7 @@ dbnode3:
- ./patroni:/patroni
env_file: docker/patroni-secrets.env
environment:
PATRONI_ETCD_URL: http://patroni_etcd:2379
PATRONI_ETCD_HOST: patroni_etcd:2379
PATRONI_NAME: dbnode3
PATRONI_SCOPE: testcluster
@@ -53,6 +53,6 @@ haproxy:
- "5000:5000"
- "5001:5001"
environment:
PATRONI_ETCD_URL: http://patroni_etcd:2379
PATRONI_ETCD_HOST: patroni_etcd:2379
PATRONI_SCOPE: testcluster
command: --confd
+1 -1
View File
@@ -84,7 +84,7 @@ function docker_run()
ETCD_CONTAINER="${PATRONI_SCOPE}_etcd"
docker_run ${ETCD_CONTAINER} ${DOCKER_IMAGE} --etcd
DOCKER_ARGS="--link=${ETCD_CONTAINER}:${ETCD_CONTAINER} -e PATRONI_SCOPE=${PATRONI_SCOPE} -e PATRONI_ETCD_URL=http://${ETCD_CONTAINER}:2379"
DOCKER_ARGS="--link=${ETCD_CONTAINER}:${ETCD_CONTAINER} -e PATRONI_SCOPE=${PATRONI_SCOPE} -e PATRONI_ETCD_HOST=${ETCD_CONTAINER}:2379"
PATRONI_ENV=$(sed 's/#.*//g' docker/patroni-secrets.env | sed -n 's/^PATRONI_.*$/-e &/p' | tr '\n' ' ')
PATRONI_VOLUME="-v $(dirname $(dirname $(realpath $0)))/patroni:/patroni"
+5 -5
View File
@@ -38,10 +38,10 @@ while getopts "$optspec" optchar; do
done
exec $CONFD zookeeper -node ${PATRONI_ZOOKEEPER_HOSTS}
else
while ! curl -s ${PATRONI_ETCD_URL}/v2/members | jq -r '.members[0].clientURLs[0]' | grep -q http; do
while ! curl -s ${PATRONI_ETCD_HOST}/v2/members | jq -r '.members[0].clientURLs[0]' | grep -q http; do
sleep 1
done
exec $CONFD etcd -node $PATRONI_ETCD_URL
exec $CONFD etcd -node $PATRONI_ETCD_HOST
fi
;;
etcd)
@@ -74,9 +74,9 @@ while getopts "$optspec" optchar; do
done
## We start an etcd
if [[ -z ${PATRONI_ETCD_URL} && -z ${PATRONI_ZOOKEEPER_HOSTS} ]]; then
if [[ -z ${PATRONI_ETCD_HOST} && -z ${PATRONI_ZOOKEEPER_HOSTS} ]]; then
etcd $ETCD_ARGS > /var/log/etcd.log 2> /var/log/etcd.err &
export PATRONI_ETCD_URL="http://127.0.0.1:2379"
export PATRONI_ETCD_HOST="127.0.0.1:2379"
fi
export PATRONI_SCOPE
@@ -108,7 +108,7 @@ __EOF__
mkdir -p "$HOME/.config/patroni"
[ -h "$HOME/.config/patroni/patronictl.yaml" ] || ln -s /patroni.yml "$HOME/.config/patroni/patronictl.yaml"
[ -z $CHEAT ] && exec python3 /patroni.py /patroni.yml
[ -z $CHEAT ] && exec python /patroni.py /patroni.yml
while true; do
sleep 60
-5
View File
@@ -5,11 +5,6 @@ Contributing guidelines
Wanna contribute to Patroni? Yay - here is how!
Chatting
--------
Just want to chat with other Patroni users? Looking for interactive troubleshooting help? Join us on channel #patroni in the `PostgreSQL Slack <https://postgres-slack.herokuapp.com/>`__.
Reporting issues
----------------
+6 -41
View File
@@ -1,5 +1,6 @@
.. _environment:
==================================
Environment Configuration Settings
==================================
@@ -11,13 +12,6 @@ Global/Universal
- **PATRONI\_NAME**: name of the node where the current instance of Patroni is running. Must be unique for the cluster.
- **PATRONI\_NAMESPACE**: path within the configuration store where Patroni will keep information about the cluster. Default value: "/service"
- **PATRONI\_SCOPE**: cluster name
- **PATRONI\_LOG\_LEVEL**: sets the general logging level. Default value is **INFO** (see `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>`_)
- **PATRONI\_LOG\_FORMAT**: sets the log formatting string. Default value is **%(asctime)s %(levelname)s: %(message)s** (see `the LogRecord attributes <https://docs.python.org/3.6/library/logging.html#logrecord-attributes>`_)
- **PATRONI\_LOG\_DATEFORMAT**: sets the datetime formatting string. (see the `formatTime() documentation <https://docs.python.org/3.6/library/logging.html#logging.Formatter.formatTime>`_)
- **PATRONI\_LOG\_DIR**: Directory to write application logs to. The directory must exist and be writable by the user executing Patroni. If you set this env variable, the application will retain 4 25MB logs by default. You can tune those retention values with `PATRONI_LOG_FILE_NUM` and `PATRONI_LOG_FILE_SIZE` (see below).
- **PATRONI\_LOG\_FILE\_NUM**: The number of application logs to retain.
- **PATRONI\_LOG\_FILE\_SIZE**: Size of patroni.log file (in bytes) that triggers a log rolling.
- **PATRONI\_LOG\_LOGGERS**: Redefine logging level per python module. Example ``PATRONI_LOG_LOGGERS="{patroni.postmaster: WARNING, urllib3: DEBUG}"``
Bootstrap configuration
-----------------------
@@ -31,32 +25,15 @@ Example: defining ``PATRONI_admin_PASSWORD=strongpasswd`` and ``PATRONI_admin_OP
Consul
------
- **PATRONI\_CONSUL\_HOST**: the host:port for the Consul endpoint.
- **PATRONI\_CONSUL\_URL**: url for the Consul, in format: http(s)://host:port
- **PATRONI\_CONSUL\_PORT**: (optional) Consul port
- **PATRONI\_CONSUL\_SCHEME**: (optional) **http** or **https**, defaults to **http**
- **PATRONI\_CONSUL\_TOKEN**: (optional) ACL token
- **PATRONI\_CONSUL\_VERIFY**: (optional) whether to verify the SSL certificate for HTTPS requests
- **PATRONI\_CONSUL\_CACERT**: (optional) The ca certificate. If present it will enable validation.
- **PATRONI\_CONSUL\_CERT**: (optional) File with the client certificate
- **PATRONI\_CONSUL\_KEY**: (optional) File with the client key. Can be empty if the key is part of certificate.
- **PATRONI\_CONSUL\_DC**: (optional) Datacenter to communicate with. By default the datacenter of the host is used.
- **PATRONI\_CONSUL\_CHECKS**: (optional) list of Consul health checks used for the session. If not specified Consul will use "serfHealth" in additional to the TTL based check created by Patroni. Additional checks, in particular the "serfHealth", may cause the leader lock to expire faster than in `ttl` seconds when the leader instance becomes unavailable.
- **PATRONI\_CONSUL\_REGISTER\_SERVICE**: (optional) whether or not to register a service with the name defined by the scope parameter and the tag master, replica or standby-leader depending on the node's role. Defaults to **false**
- **PATRONI\_CONSUL\_SERVICE\_CHECK\_INTERVAL**: (optional) how often to perform health check against registered url
Etcd
----
- **PATRONI\_ETCD\_PROXY**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **PATRONI\_ETCD\_URL**
- **PATRONI\_ETCD\_URL**: url for the etcd, in format: http(s)://(username:password@)host:port
- **PATRONI\_ETCD\_HOSTS**: list of etcd endpoints in format 'host1:port1','host2:port2',etc...
- **PATRONI\_ETCD\_PROTOCOL**: http or https, if not specified http is used. If the **url** or **proxy** is specified - will take protocol from them.
- **PATRONI\_ETCD\_HOST**: the host:port for the etcd endpoint.
- **PATRONI\_ETCD\_URL**: url for the etcd, in format: http(s)://(username:password@)host:port
- **PATRONI\_ETCD\_PROXY**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **PATRONI\_ETCD\_URL**
- **PATRONI\_ETCD\_SRV**: Domain to search the SRV record(s) for cluster autodiscovery.
- **PATRONI\_ETCD\_USERNAME**: username for etcd authentication.
- **PATRONI\_ETCD\_PASSWORD**: password for etcd authentication.
- **PATRONI\_ETCD\_CACERT**: The ca certificate. If present it will enable validation.
- **PATRONI\_ETCD\_CERT**: File with the client certificate.
- **PATRONI\_ETCD\_CACERT**: The ca certificate. If pressent it will enable validation.
- **PATRONI\_ETCD\_CERT**: File with the client certificate
- **PATRONI\_ETCD\_KEY**: File with the client key. Can be empty if the key is part of certificate.
Exhibitor
@@ -64,18 +41,6 @@ Exhibitor
- **PATRONI\_EXHIBITOR\_HOSTS**: initial list of Exhibitor (ZooKeeper) nodes in format: 'host1,host2,etc...'. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.
- **PATRONI\_EXHIBITOR\_PORT**: Exhibitor port.
.. _kubernetes_environment:
Kubernetes
----------
- **PATRONI\_KUBERNETES\_NAMESPACE**: (optional) Kubernetes namespace where the Patroni pod is running. Default value is `default`.
- **PATRONI\_KUBERNETES\_LABELS**: Labels in format ``{label1: value1, label2: value2}``. These labels will be used to find existing objects (Pods and either Endpoints or ConfigMaps) associated with the current cluster. Also Patroni will set them on every object (Endpoint or ConfigMap) it creates.
- **PATRONI\_KUBERNETES\_SCOPE\_LABEL**: (optional) name of the label containing cluster name. Default value is `cluster-name`.
- **PATRONI\_KUBERNETES\_ROLE\_LABEL**: (optional) name of the label containing Postgres role (`master` or `replica`). Patroni will set this label on the pod it is running in. Default value is `role`.
- **PATRONI\_KUBERNETES\_USE\_ENDPOINTS**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
- **PATRONI\_KUBERNETES\_POD\_IP**: (optional) IP address of the pod Patroni is running in. This value is required when `PATRONI_KUBERNETES_USE_ENDPOINTS` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
- **PATRONI\_KUBERNETES\_PORTS**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``PATRONI_KUBERNETES_PORTS='{[{"name": "postgresql", "port": 5432}]}'`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `PATRONI_KUBERNETES_USE_ENDPOINTS` is set.
PostgreSQL
----------
- **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
@@ -90,7 +55,7 @@ PostgreSQL
- **PATRONI\_SUPERUSER\_PASSWORD**: password for the superuser, set during initialization (initdb).
REST API
--------
--------
- **PATRONI\_RESTAPI\_CONNECT\_ADDRESS**: IP address and port to access the REST API.
- **PATRONI\_RESTAPI\_LISTEN**: IP address and port that Patroni will listen to, to provide health-check information for HAProxy.
- **PATRONI\_RESTAPI\_USERNAME**: Basic-auth username to protect unsafe REST API endpoints.
+19 -56
View File
@@ -1,8 +1,8 @@
.. _readme:
============
Introduction
============
=================
How Patroni Works
=================
Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features.
@@ -13,67 +13,30 @@ For additional background info, see:
* `PostgreSQL HA with Kubernetes and Patroni <https://www.youtube.com/watch?v=iruaCgeG7qs>`__, talk by Josh Berkus at KubeCon 2016 (video)
* `Feb. 2016 Zalando Tech blog post <https://tech.zalando.de/blog/zalandos-patroni-a-template-for-high-availability-postgresql/>`__
==================
Development Status
------------------
==================
Patroni is in active development and accepts contributions. See our :ref:`Contributing <contributing>` section below for more details.
We report new releases information :ref:`here <releases>`.
===================================
Technical Requirements/Installation
-----------------------------------
===================================
**Pre-requirements for Mac OS**
**For Mac**
To install requirements on a Mac, run the following:
::
brew install postgresql etcd haproxy libyaml python
pip install psycopg2 pyyaml
**General installation for pip**
Patroni can be installed with pip:
::
pip install patroni[dependencies]
where dependencies can be either empty, or consist of one or more of the following:
etcd
`python-etcd` module in order to use Etcd as DCS
consul
`python-consul` module in order to use Consul as DCS
zookeeper
`kazoo` module in order to use Zookeeper as DCS
exhibitor
`kazoo` module in order to use Exhibitor as DCS (same dependencies as for Zookeeper)
kubernetes
`kubernetes` module in order to use Kubernetes as DCS in Patroni
aws
`boto` in order to use AWS callbacks
For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is:
::
pip install patroni[etcd,aws]
Note that external tools to call in the replica creation or custom bootstrap scripts (i.e. WAL-E) should be installed
independently of Patroni.
.. _running_configuring:
=======================
Running and Configuring
-----------------------
The following section assumes Patroni repository as being cloned from https://github.com/zalando/patroni. Namely, you
will need example configuration files `postgres0.yml` and `postgres1.yml`. If you installed Patroni with pip, you can
obtain those files from the git repository and replace `./patroni.py` below with `patroni` command.
=======================
To get started, do the following from different terminals:
::
@@ -97,27 +60,27 @@ run:
> psql --host 127.0.0.1 --port 5000 postgres
==================
YAML Configuration
------------------
==================
Go :ref:`here <settings>` for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
=========================
Environment Configuration
-------------------------
=========================
Go :ref:`here <environment>` for comprehensive information about configuring(overriding) settings via environment variables.
===================
Replication Choices
-------------------
===================
Patroni uses Postgres' streaming replication, which is asynchronous by default. Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the leader. This setting should be increased or decreased based on business requirements. It's also possible to use synchronous replication for better durability guarantees. See :ref:`replication modes documentation <replication_modes>` for details.
======================================
Applications Should Not Use Superusers
--------------------------------------
======================================
When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
+19 -89
View File
@@ -10,20 +10,6 @@ Global/Universal
- **namespace**: path within the configuration store where Patroni will keep information about the cluster. Default value: "/service"
- **scope**: cluster name
Log
---
- **level**: sets the general logging level. Default value is **INFO** (see `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>`_)
- **format**: sets the log formatting string. Default value is **%(asctime)s %(levelname)s: %(message)s** (see `the LogRecord attributes <https://docs.python.org/3.6/library/logging.html#logrecord-attributes>`_)
- **dateformat**: sets the datetime formatting string. (see the `formatTime() documentation <https://docs.python.org/3.6/library/logging.html#logging.Formatter.formatTime>`_)
- **dir**: Directory to write application logs to. The directory must exist and be writable by the user executing Patroni. If you set this value, the application will retain 4 25MB logs by default. You can tune those retention values with `file_num` and `file_size` (see below).
- **file\_num**: The number of application logs to retain.
- **file\_size**: Size of patroni.log file (in bytes) that triggers a log rolling.
- **loggers**: This section allows redefining logging level per python module
- **patroni.postmaster: WARNING**
- **urllib3: DEBUG**
.. _bootstrap_settings:
Bootstrap configuration
-----------------------
- **dcs**: This section will be written into `/<namespace>/<scope>/config` of a given configuration store after initializing of new cluster. This is the global configuration for the cluster. If you want to change some parameters for all cluster nodes - just do it in DCS (or via Patroni API) and all nodes will apply this configuration.
@@ -32,27 +18,13 @@ Bootstrap configuration
- **retry\_timeout**: timeout for DCS and PostgreSQL operation retries. DCS or network issues shorter than this will not cause Patroni to demote the leader. Default value: 10
- **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election.
- **master\_start\_timeout**: the amount of time a master is allowed to recover from failures before failover is triggered. Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Best worst case failover time for master failure is: loop\_wait + master\_start\_timeout + loop\_wait, unless master\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff.
- **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that successfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See :ref:`replication modes documentation <replication_modes>` for details.
- **synchronous\_mode\_strict**: prevents disabling synchronous replication if no synchronous replicas are available, blocking all client writes to the master. See :ref:`replication modes documentation <replication_modes>` for details.
- **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that succesfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See `replication modes documentation <https://github.com/zalando/patroni/blob/master/docs/replication_modes.rst>`__ for details.
- **postgresql**:
- **use\_pg\_rewind**: whether or not to use pg_rewind
- **use\_pg\_rewind**:whether or not to use pg_rewind
- **use\_slots**: whether or not to use replication_slots. Must be False for PostgreSQL 9.3. You should comment out max_replication_slots before it becomes ineligible for leader status.
- **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower.
- **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower.
- **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.
- **standby\_cluster**: if this section is defined, we want to bootstrap a standby cluster.
- **host**: an address of remote master
- **port**: a port of remote master
- **primary\_slot\_name**: which slot on the remote master to use for replication. This parameter is optional, the default value is derived from the instance name (see function `slot_name_from_member_name`).
- **create\_replica\_methods**: an ordered list of methods that can be used to bootstrap standby leader from the remote master, can be different from the list defined in :ref:`postgresql_settings`
- **restore\_command**: command to restore WAL records from the remote master to standby leader, can be different from the list defined in :ref:`postgresql_settings`
- **archive\_cleanup\_command**: cleanup command for standby leader
- **recovery\_min\_apply\_delay**: how long to wait before actually apply WAL records on a standby leader
- **slots**: define permanent replication slots. These slots will be preserved during switchover/failover. Patroni will try to create slots before opening connections to the cluster.
- **my_slot_name**: the name of replication slot. It is the responsibility of the operator to make sure that there are no clashes in names between replication slots automatically created by Patroni for members and permanent replication slots.
- **type**: slot type. Could be ``physical`` or ``logical``. If the slot is logical, you have to additionally define ``database`` and ``plugin``.
**database**: the database name where logical slots should be created.
**plugin**: the plugin name for the logical slot.
- **method**: custom script to use for bootstrapping this cluster.
- **method**: custom script to use for bootstrpapping this cluster.
See :ref:`custom bootstrap methods documentation <custom_bootstrap>` for details.
When ``initdb`` is specified revert to the default ``initdb`` command. ``initdb`` is also triggered when no ``method``
parameter is present in the configuration file.
@@ -63,7 +35,7 @@ Bootstrap configuration
- **pg\_hba**: list of lines that you should add to pg\_hba.conf.
- **- host all all 0.0.0.0/0 md5**.
- **- host replication replicator 127.0.0.1/32 md5**: A line like this is required for replication.
- **users**: Some additional users which need to be created after initializing new cluster
- **users**: Some additional users users which needs to be created after initializing new cluster
- **admin**: the name of user
- **password: zalando**:
- **options**: list of options for CREATE USER statement
@@ -71,40 +43,22 @@ Bootstrap configuration
- **- createdb**
- **post\_bootstrap** or **post\_init**: An additional script that will be executed after initializing the cluster. The script receives a connection string URL (with the cluster superuser as a user name). The PGPASSFILE variable is set to the location of pgpass file.
.. _consul_settings:
Consul
------
Most of the parameters are optional, but you have to specify one of the **host** or **url**
- **host**: the host:port for the Consul endpoint, in format: http(s)://host:port
- **url**: url for the Consul endpoint
- **port**: (optional) Consul port
- **scheme**: (optional) **http** or **https**, defaults to **http**
- **token**: (optional) ACL token
- **verify**: (optional) whether to verify the SSL certificate for HTTPS requests
- **cacert**: (optional) The ca certificate. If present it will enable validation.
- **cert**: (optional) file with the client certificate
- **key**: (optional) file with the client key. Can be empty if the key is part of **cert**.
- **dc**: (optional) Datacenter to communicate with. By default the datacenter of the host is used.
- **checks**: (optional) list of Consul health checks used for the session. If not specified Consul will use "serfHealth" in additional to the TTL based check created by Patroni. Additional checks, in particular the "serfHealth", may cause the leader lock to expire faster than in `ttl` seconds when the leader instance becomes unavailable
- **register\_service**: (optional) whether or not to register a service with the name defined by the scope parameter and the tag master, replica or standby-leader depending on the node's role. Defaults to **false**
- **service\_check\_interval**: (optional) how often to perform health check against registered url
- **host**: the host:port for the Consul endpoint.
Etcd
----
Most of the parameters are optional, but you have to specify one of the **host**, **hosts**, **url**, **proxy** or **srv**
Most of the parameters are optional, but you have to specify one of the **host**, **url**, **proxy** or **srv**
- **host**: the host:port for the etcd endpoint.
- **hosts**: list of etcd endpoint in format host1:port1,host2:port2,etc... Could be a comma separated string or an actual yaml list.
- **url**: url for the etcd
- **proxy**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **url**
- **srv**: Domain to search the SRV record(s) for cluster autodiscovery.
- **protocol**: (optional) http or https, if not specified http is used. If the **url** or **proxy** is specified - will take protocol from them.
- **username**: (optional) username for etcd authentication.
- **username**: (optional) username for etcd authentication
- **password**: (optional) password for etcd authentication.
- **cacert**: (optional) The ca certificate. If present it will enable validation.
- **cert**: (optional) file with the client certificate.
- **cacert**: (optional) The ca certificate. If pressent it will enable validation.
- **cert**: (optional) file with the client certificate
- **key**: (optional) file with the client key. Can be empty if the key is part of **cert**.
Exhibitor
@@ -113,18 +67,6 @@ Exhibitor
- **poll\_interval**: how often the list of ZooKeeper and Exhibitor nodes should be updated from Exhibitor
- **port**: Exhibitor port.
.. _kubernetes_settings:
Kubernetes
----------
- **namespace**: (optional) Kubernetes namespace where Patroni pod is running. Default value is `default`.
- **labels**: Labels in format ``{label1: value1, label2: value2}``. These labels will be used to find existing objects (Pods and either Endpoints or ConfigMaps) associated with the current cluster. Also Patroni will set them on every object (Endpoint or ConfigMap) it creates.
- **scope\_label**: (optional) name of the label containing cluster name. Default value is `cluster-name`.
- **role\_label**: (optional) name of the label containing role (master or replica). Patroni will set this label on the pod it runs in. Default value is ``role``.
- **use\_endpoints**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
- **pod\_ip**: (optional) IP address of the pod Patroni is running in. This value is required when `use_endpoints` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
- **ports**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``kubernetes.ports: {[{"name": "postgresql", "port": 5432}]}`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `kubernetes.use_endpoints` is set.
.. _postgresql_settings:
PostgreSQL
@@ -143,17 +85,17 @@ PostgreSQL
- **on\_start**: run this script when the cluster starts.
- **on\_stop**: run this script when the cluster stops.
- **connect\_address**: IP address + port through which Postgres is accessible from other nodes and applications.
- **create\_replica\_methods**: an ordered list of the create methods for turning a Patroni node into a new replica.
- **create\_replica\_method**: an ordered list of the create methods for turning a Patroni node into a new replica.
"basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its
own config item. See :ref:`custom replica creation methods documentation <custom_replica_creation>` for further explanation.
- **data\_dir**: The location of the Postgres data directory, either :ref:`existing <existing_data>` or to be initialized by Patroni.
- **data\_dir**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
- **config\_dir**: The location of the Postgres configuration directory, defaults to the data directory. Must be writable by Patroni.
- **bin\_dir**: Path to PostgreSQL binaries (pg_ctl, pg_rewind, pg_basebackup, postgres). The default value is an empty string meaning that PATH environment variable will be used to find the executables.
- **bin\_dir**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The default value is an empty string meaning that PATH environment variable will be used to find the executables.
- **listen**: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
- **use\_unix\_socket**: specifies that Patroni should prefer to use unix sockets to connect to the cluster. Default value is ``false``. If ``unix_socket_directories`` is definded, Patroni will use first suitable value from it to connect to the cluster and fallback to tcp if nothing is suitable. If ``unix_socket_directories`` is not specified in ``postgresql.parameters``, Patroni will assume that default value should be used and omit ``host`` from connection parameters.
- **pgpass**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup, the post_init script and under some other circumstances. The location must be writable by Patroni.
- **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower.
- **custom\_conf** : path to an optional custom ``postgresql.conf`` file, that will be used in place of ``postgresql.base.conf``. The file must exist on all cluster nodes, be readable by PostgreSQL and will be included from its location on the real ``postgresql.conf``. Note that Patroni will not monitor this file for changes, nor backup it. However, its settings can still be overridden by Patroni's own configuration facilities - see :ref:`dynamic configuration <dynamic_configuration>` for details.
- **custom\_conf** : path to an optional custom ``postgresql.conf`` file, that will be used in place of ``postgresql.base.conf``. The file must exist on all cluster nodes, be readable by PostgreSQL and will be included from its location on the real ``postgresql.conf``. Note that Patroni will not monitor this file for changes, nor backup it. However, its settings can still be overriden by Patroni's own configuration facilities - see `dynamic configuration <https://github.com/zalando/patroni/blob/master/docs/dynamic_configuration.rst>`__ for details.
- **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.
- **pg\_hba**: list of lines that Patroni will use to generate ``pg_hba.conf``. This parameter has higher priority than ``bootstrap.pg_hba``. Together with :ref:`dynamic configuration <dynamic_configuration>` it simplifies management of ``pg_hba.conf``.
- **- host all all 0.0.0.0/0 md5**.
@@ -161,15 +103,12 @@ PostgreSQL
- **pg\_ctl\_timeout**: How long should pg_ctl wait when doing ``start``, ``stop`` or ``restart``. Default value is 60 seconds.
- **use\_pg\_rewind**: try to use pg\_rewind on the former leader when it joins cluster as a replica.
- **remove\_data\_directory\_on\_rewind\_failure**: If this option is enabled, Patroni will remove postgres data directory and recreate replica. Otherwise it will try to follow the new leader. Default value is **false**.
- **remove\_data\_directory\_on\_diverged\_timelines**: Patroni will remove postgres data directory and recreate replica if it notices that timelines are diverging and the former master can not start streaming from the new master. This option is useful when ``pg_rewind`` can not be used. Default value is **false**.
- **replica\_method**: for each create_replica_methods other than basebackup, you would add a configuration section of the same name. At a minimum, this should include "command" with a full path to the actual script to be executed. Other configuration parameters will be passed along to the script in the form "parameter=value".
- **replica\_method** for each create_replica_method other than basebackup, you would add a configuration section of the same name. At a minimum, this should include "command" with a full path to the actual script to be executed. Other configuration parameters will be passed along to the script in the form "parameter=value".
REST API
--------
- **connect\_address**: IP address (or hostname) and port, to access the Patroni's REST API. All the members of the cluster must be able to connect to this address, so unless the Patroni setup is intended for a demo inside the localhost, this address must be a non "localhost" or loopback addres (ie: "localhost" or "127.0.0.1"). It can serve as a endpoint for HTTP health checks (read below about the "listen" REST API parameter), and also for user queries (either directly or via the REST API), as well as for the health checks done by the cluster members during leader elections (for example, to determine whether the master is still running, or if there is a node which has a WAL position that is ahead of the one doing the query; etc.) The connect_address is put in the member key in DCS, making it possible to translate the member name into the address to connect to its REST API.
- **listen**: IP address (or hostname) and port that Patroni will listen to for the REST API - to provide also the same health checks and cluster messaging between the participating nodes, as described above. to provide health-check information for HAProxy (or any other load balancer capable of doing a HTTP "OPTION" or "GET" checks).
--------
- **connect\_address**: IP address and port to access the REST API.
- **listen**: IP address and port that Patroni will listen to, to provide health-check information for HAProxy.
- **Optional**:
- **authentication**:
- **username**: Basic-auth username to protect unsafe REST API endpoints.
@@ -178,21 +117,12 @@ REST API
- **certfile**: Specifies the file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL.
- **keyfile**: Specifies the file with the secret key in the PEM format.
.. _patronictl_settings:
CTL
---
- **Optional**:
- **insecure**: Allow connections to REST API without verifying SSL certs.
- **cacert**: Specifices the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs.
- **certfile**: Specifies the file with the certificate in the PEM format to use while verifying REST API SSL certs. If not provided patronictl will use the value provided for REST API "certfile" parameter.
ZooKeeper
----------
- **hosts**: list of ZooKeeper cluster members in format: ['host1:port1', 'host2:port2', 'etc...'].
Watchdog
--------
- **mode**: ``off``, ``automatic`` or ``required``. When ``off`` watchdog is disabled. When ``automatic`` watchdog will be used if available, but ignored if it is not. When ``required`` the node will not become a leader unless watchdog can be successfully enabled.
- **mode**: ``off``, ``automatic`` or ``required``. When ``off`` watchdog is disabled. When ``automatic`` watchdog will be used if available, but ignored if it is not. When ``required`` the node will not become a leader unless watchdog can be succesfully enabled.
- **device**: Path to watchdog device. Defaults to ``/dev/watchdog``.
- **safety_margin**: Number of seconds of safety margin between watchdog triggering and leader key expiration.
+5 -7
View File
@@ -18,11 +18,9 @@
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
import sys
sys.path.insert(0, os.path.abspath('..'))
from patroni.version import __version__
# -- General configuration ------------------------------------------------
@@ -53,7 +51,7 @@ master_doc = 'index'
# General information about the project.
project = 'Patroni'
copyright = '2015 Compose, Zalando SE'
copyright = '2016, Zalando SE'
author = 'Zalando SE'
# The version info for the project you're documenting, acts as replacement for
@@ -61,9 +59,9 @@ author = 'Zalando SE'
# built documents.
#
# The short X.Y version.
version = __version__[:__version__.rfind('.')]
version = '1.2'
# The full version, including alpha/beta/rc tags.
release = __version__
release = '1.2.2'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
+3 -4
View File
@@ -16,9 +16,9 @@ Patroni configuration is stored in the DCS (Distributed Configuration Store). Th
These options are defined in the configuration file and take precedence over dynamic configuration.
patroni.yml could be changed and reload in runtime (without restart of Patroni) by sending SIGHUP to the Patroni process or by performing ``POST /reload`` REST-API request.
- Environment :ref:`configuration <environment>`.
- Environment :ref:`configuration <environment>` .
It is possible to set/override some of the "Local" configuration parameters with environment variables.
Environment configuration is very useful when you are running in a dynamic environment and you don't know some of the parameters in advance (for example it's not possible to know your external IP address when you are running inside ``docker``).
Environment configuration is very useful when you are running in a dynamic environment and you don't know some of the parameters in advance (for example it's not possible to know you external IP address when you are running inside ``docker``).
Some of the PostgreSQL parameters must hold the same values on the master and the replicas. For those, values set either in the local patroni configuration files or via the environment variables take no effect. To alter or set their values one must change the shared configuration in the DCS. Below is the actual list of such parameters together with the default values:
@@ -42,7 +42,7 @@ There are some other Postgres parameters controlled by Patroni:
- listen_addresses - is set either from ``postgresql.listen`` or from ``PATRONI_POSTGRESQL_LISTEN`` environment variable
- port - is set either from ``postgresql.listen`` or from ``PATRONI_POSTGRESQL_LISTEN`` environment variable
- cluster_name - is set either from ``scope`` or from ``PATRONI_SCOPE`` environment variable
- cluster_name - is set either from ``scope`` or from ``PATRRONI_SCOPE`` environment variable
- hot_standby: on
To be on the safe side parameters from the above lists are not written into ``postgresql.conf``, but passed as a list of arguments to the ``pg_ctl start`` which gives them the highest precedence, even above `ALTER SYSTEM <https://www.postgresql.org/docs/current/static/sql-altersystem.html>`__
@@ -76,7 +76,6 @@ Also, the following Patroni configuration options can be changed only dynamicall
- loop_wait: 10
- retry_timeouts: 10
- maximum_lag_on_failover: 1048576
- check_timeline: false
- postgresql.use_slots: true
Upon changing these options, Patroni will read the relevant section of the configuration stored in DCS and change its
-35
View File
@@ -1,35 +0,0 @@
.. _existing_data:
Convert a Standalone to a Patroni Cluster
=========================================
This section describes the process for converting a standalone PostgreSQL instance into a Patroni cluster.
To deploy a Patroni cluster without using a pre-existing PostgreSQL instance, see :ref:`Running and Configuring <running_configuring>` instead.
Procedure
---------
A Patroni cluster can be started with a data directory from a single-node PostgreSQL database. This is achieved by following closely these steps:
#. Manually start PostgreSQL daemon
#. Create Patroni superuser and replication users as defined in the :ref:`authentication <postgresql_settings>` section of the Patroni configuration. If this user is created in SQL, the following queries achieve this:
.. code-block:: sql
CREATE USER $PATRONI_SUPERUSER_USERNAME WITH SUPERUSER ENCRYPTED PASSWORD '$PATRONI_SUPERUSER_PASSWORD';
CREATE USER $PATRONI_REPLICATION_USERNAME WITH REPLICATION ENCRYPTED PASSWORD '$PATRONI_REPLICATION_PASSWORD';
#. Start Patroni (e.g. ``patroni /etc/patroni/patroni.yml``). It automatically detects that PostgreSQL daemon is already running but its configuration might be out-of-date.
#. Ask Patroni to restart the node with ``patronictl restart cluster-name node-name``.
FAQ
---
#. During Patroni startup, Patroni complains that it cannot bind to the PostgreSQL port.
You need to verify ``listen_addresses`` and ``port`` in ``postgresql.conf`` and ``postgresql.listen`` in ``patroni.yml``. Don't forget that ``pg_hba.conf`` should allow such access.
#. After asking Patroni to restart the node, PostgreSQL displays the error message ``could not open configuration file "/etc/postgresql/10/main/pg_hba.conf": No such file or directory``
It can mean various things depending on how you manage PostgreSQL configuration. If you specified `postgresql.config_dir`, Patroni generates the ``pg_hba.conf`` based on the settings in the :ref:`bootstrap <bootstrap_settings>` section only when it bootstraps a new cluster. In this scenario the ``PGDATA`` was not empty, therefore no bootstrap happened. This file must exist beforehand.
+4 -4
View File
@@ -6,11 +6,11 @@
Introduction
============
Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__, `Consul <https://github.com/hashicorp/consul>`__ or `Kubernetes <https://kubernetes.io>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely. There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
**Note to Kubernetes users**: Patroni can run natively on top of Kubernetes. Take a look at the :ref:`Kubernetes <kubernetes>` chapter of the Patroni documentation.
**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. To this end, we've created a `Helm Chart <https://github.com/kubernetes/charts/tree/master/incubator/patroni>`__ that enables you to deploy a five-node Patroni cluster using a Kubernetes PetSet.
.. toctree::
@@ -24,8 +24,6 @@ We call Patroni a "template" because it is far from being a one-size-fits-all or
replica_bootstrap
replication_modes
pause
kubernetes
watchdog
releases
CONTRIBUTING
@@ -35,3 +33,5 @@ Indices and tables
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
-53
View File
@@ -1,53 +0,0 @@
.. _kubernetes:
Using Patroni with Kubernetes
=============================
Patroni can use Kubernetes objects in order to store the state of the cluster and manage the leader key. That makes it
capable of operating Postgres in Kubernetes environment without any consistency store, namely, one doesn't
need to run an extra Etcd deployment. There are two different type of Kubernetes objects Patroni can use to store the
leader and the configuration keys, they are configured with the `kubernetes.use_endpoints` or `PATRONI_KUBERNETES_USE_ENDPOINTS`
environment variable.
Use Endpoints
-------------
Despite the fact that this is the recommended mode, it is turned off by default for compatibility reasons. When it is on, Patroni stores
the cluster configuration and the leader key in the `metadata: annotations` fields of the respective `Endpoints` it creates.
Changing the leader is safer than when using `ConfigMaps`, since both the annotations, containing the leader information, and the actual addresses
pointing to the running leader pod are updated simultaneously in one go.
Use ConfigMaps
--------------
In this mode, Patroni will create ConfigMaps instead of Endpoints and store keys inside meta-data of those ConfigMaps.
Changing the leader takes at least two updates, one to the leader ConfigMap and another to the respective Endpoint.
There are two ways to direct the traffic to the Postgres master:
- use the `callback script <https://github.com/zalando/patroni/blob/master/kubernetes/callback.py>`_ provided by Patroni
- configure the Kubernetes Postgres service to use the label selector with the `role_label` (configured in patroni configuration).
Note that in some cases, for instance, when running on OpenShift, there is no alternative to using ConfigMaps.
Configuration
-------------
Patroni Kubernetes :ref:`settings <kubernetes_settings>` and :ref:`environment variables <kubernetes_environment>` are described in the general chapters of the documentation.
Examples
--------
- The `kubernetes <https://github.com/zalando/patroni/tree/master/kubernetes>`__ folder of the Patroni repository contains
examples of the Docker image, the Kubernetes manifest and the callback script in order to test Patroni Kubernetes setup.
Note that in the current state it will not be able to use PersistentVolumes because of permission issues.
- You can find the full-featured Docker image that can use Persistent Volumes in the
`Spilo Project <https://github.com/zalando/spilo>`_.
- There is also a `Helm chart <https://github.com/kubernetes/charts/tree/master/incubator/patroni>`_
to deploy the Spilo image configured with Patroni running using Kubernetes.
- In order to run your database clusters at scale using Patroni and Spilo, take a look at the
`postgres-operator <https://github.com/zalando-incubator/postgres-operator>`_ project. It implements the operator pattern
to manage Spilo clusters.
+1 -1
View File
@@ -25,7 +25,7 @@ When Patroni runs in a paused mode, it does not change the state of PostgreSQL,
- If there is no leader lock in the cluster, the running master acquires the lock. If there is more than one master node, then the first master to acquire the lock wins. If there are no masters altogether, Patroni does not try to promote any replicas. There is an exception in this rule: if there is no leader lock because the old master has demoted itself due to the manual promotion, then only the candidate node mentioned in the promotion request may take the leader lock. When the new leader lock is granted (i.e. after promoting a replica manually), Patroni makes sure the replicas that were streaming from the previous leader will switch to the new one.
- When Postgres is stopped, Patroni does not try to start it. When Patroni is stopped, it does not try to stop the Postgres instance it is managing.
- When Postgres is stopped, Patroni does not try to start it. When Patroni is stopped, it does not to stop Postgres instance it is managing.
User guide
----------
-644
View File
@@ -3,650 +3,6 @@
Release notes
=============
Version 1.5.5
-------------
This version introduces the possibility of automatic reinit of the former master, improves patronictl list output and fixes a number of bugs.
**New features**
- Add support of `PATRONI_ETCD_PROTOCOL`, `PATRONI_ETCD_USERNAME` and `PATRONI_ETCD_PASSWORD` environment variables (Étienne M)
Before it was possible to configure them only in the config file or as a part of `PATRONI_ETCD_URL`, which is not always convenient.
- Make it possible to automatically reinit the former master (Alexander Kukushkin)
If the pg_rewind is disabled or can't be used, the former master could fail to start as a new replica due to diverged timelines. In this case, the only way to fix it is wiping the data directory and reinitializing. This behavior could be changed by setting `postgresql.remove_data_directory_on_diverged_timelines`. When it is set, Patroni will wipe the data directory and reinitialize the former master automatically.
- Show information about timelines in patronictl list (Alexander)
It helps to detect stale replicas. In addition to that, `Host` will include ':{port}' if the port value isn't default or there is more than one member running on the same host.
- Create a headless service associated with the $SCOPE-config endpoint (Alexander)
The "config" endpoint keeps information about the cluster-wide Patroni and Postgres configuration, history file, and last but the most important, it holds the `initialize` key. When the Kubernetes master node is restarted or upgraded, it removes endpoints without services. The headless service will prevent it from being removed.
**Bug fixes**
- Adjust the read timeout for the leader watch blocking query (Alexander)
According to the Consul documentation, the actual response timeout is increased by a small random amount of additional wait time added to the supplied maximum wait time to spread out the wake up time of any concurrent requests. It adds up to `wait / 16` additional time to the maximum duration. In our case we are adding `wait / 15` or 1 second depending on what is bigger.
- Always use replication=1 when connecting via replication protocol to the postgres (Alexander)
Starting from Postgres 10 the line in the pg_hba.conf with database=replication doesn't accept connections with the parameter replication=database.
- Don't write primary_conninfo into recovery.conf for wal-only standby cluster (Alexander)
Despite not having neither `host` nor `port` defined in the `standby_cluster` config, Patroni was putting the `primary_conninfo` into the `recovery.conf`, which is useless and generating a lot of errors.
Version 1.5.4
-------------
This version implements flexible logging and fixes a number of bugs.
**New features**
- Improvements in logging infrastructure (Alexander Kukushkin, Lucas Capistrant, Alexander Anikin)
Logging configuration could be configured not only from environment variables but also from Patroni config file. It makes it possible to change logging configuration in runtime by updating config and doing reload or sending SIGHUP to the Patroni process. By default Patroni writes logs to stderr, but now it becomes possible to write logs directly into the file and rotate when it reaches a certain size. In addition to that added support of custom dateformat and the possibility to fine-tune log level for each python module.
- Make it possible to take into account the current timeline during leader elections (Alexander Kukushkin)
It could happen that the node is considering itself as a healthiest one although it is currently not on the latest known timeline. In some cases we want to avoid promoting of such node, which could be achieved by setting `check_timeline` parameter to `true` (default behavior remains unchanged).
- Relaxed requirements on superuser credentials
Libpq allows opening connections without explicitly specifying neither username nor password. Depending on situation it relies either on pgpass file or trust authentication method in pg_hba.conf. Since pg_rewind is also using libpq, it will work the same way.
- Implemented possibility to configure Consul Service registration and check interval via environment variables (Alexander Kukushkin)
Registration of service in Consul was added in the 1.5.0, but so far it was only possible to turn it on via patroni.yaml.
**Stability Improvements**
- Set archive_mode to off during the custom bootstrap (Alexander Kukushkin)
We want to avoid archiving wals and history files until the cluster is fully functional. It really helps if the custom bootstrap involves pg_upgrade.
- Apply five seconds backoff when loading global config on start (Alexander Kukushkin)
It helps to avoid hammering DCS when Patroni just starting up.
- Reduce amount of error messages generated on shutdown (Alexander Kukushkin)
They were harmless but rather annoying and sometimes scary.
- Explicitly secure rw perms for recovery.conf at creation time (Lucas)
We don't want anybody except patroni/postgres user reading this file, because it contains replication user and password.
- Redirect HTTPServer exceptions to logger (Julien Riou)
By default, such exceptions were logged on standard output messing with regular logs.
**Bug fixes**
- Removed stderr pipe to stdout on pg_ctl process (Cody Coons)
Inheriting stderr from the main Patroni process allows all Postgres logs to be seen along with all patroni logs. This is very useful in a container environment as Patroni and Postgres logs may be consumed using standard tools (docker logs, kubectl, etc). In addition to that, this change fixes a bug with Patroni not being able to catch postmaster pid when postgres writing some warnings into stderr.
- Set Consul service check deregister timeout in Go time format (Pavel Kirillov)
Without explicitly mentioned time unit registration was failing.
- Relax checks of standby_cluster cluster configuration (Dmitry Dolgov, Alexander Kukushkin)
It was accepting only strings as valid values and therefore it was not possible to specify the port as integer and create_replica_methods as a list.
Version 1.5.3
-------------
Compatibility and bugfix release.
- Improve stability when running with python3 against zookeeper (Alexander Kukushkin)
Change of `loop_wait` was causing Patroni to disconnect from zookeeper and never reconnect back.
- Fix broken compatibility with postgres 9.3 (Alexander)
When opening a replication connection we should specify replication=1, beacuse 9.3 does not understand replication='database'
- Make sure we refresh Consul session at least once per HA loop and improve handling of consul sessions exceptions (Alexander)
Restart of local consul agent invalidates all sessions related to the node. Not calling session refresh on time and not doing proper handling of session errors was causing demote of the primary.
Version 1.5.2
-------------
Compatibility and bugfix release.
- Compatibility with kazoo-2.6.0 (Alexander Kukushkin)
In order to make sure that requests are performed with an appropriate timeout, Patroni redefines create_connection method from python-kazoo module. The last release of kazoo slightly changed the way how create_connection method is called.
- Fix Patroni crash when Consul cluster loses the leader (Alexander)
The crash was happening due to incorrect implementation of touch_member method, it should return boolean and not raise any exceptions.
Version 1.5.1
-------------
This version implements support of permanent replication slots, adds support of pgBackRest and fixes number of bugs.
**New features**
- Permanent replication slots (Alexander Kukushkin)
Permanent replication slots are preserved on failover/switchover, that is, Patroni on the new primary will create configured replication slots right after doing promote. Slots could be configured with the help of `patronictl edit-config`. The initial configuration could be also done in the :ref:`bootstrap.dcs <settings>`.
- Add pgbackrest support (Yogesh Sharma)
pgBackrest can restore in existing $PGDATA folder, this allows speedy restore as files which have not changed since last backup are skipped, to support this feature new parameter `keep_data` has been introduced. See :ref:`replica creation method <custom_replica_creation>` section for additional examples.
**Bug fixes**
- A few bugfixes in the "standby cluster" workflow (Alexander)
Please see https://github.com/zalando/patroni/pull/823 for more details.
- Fix REST API health check when cluster management is paused and DCS is not accessible (Alexander)
Regression was introduced in https://github.com/zalando/patroni/commit/90cf930036a9d5249265af15d2b787ec7517cf57
Version 1.5.0
-------------
This version enables Patroni HA cluster to operate in a standby mode, introduces experimental support for running on Windows, and provides a new configuration parameter to register PostgreSQL service in Consul.
**New features**
- Standby cluster (Dmitry Dolgov)
One or more Patroni nodes can form a standby cluster that runs alongside the primary one (i.e. in another datacenter) and consists of standby nodes that replicate from the master in the primary cluster. All PostgreSQL nodes in the standby cluster are replicas; one of those replicas elects itself to replicate directly from the remote master, while the others replicate from it in a cascading manner. More detailed description of this feature and some configuration examples can be found at :ref:`here <standby_cluster>`.
- Register Services in Consul (Pavel Kirillov, Alexander Kukushkin)
If `register_service` parameter in the consul :ref:`configuration <consul_settings>` is enabled, the node will register a service with the name `scope` and the tag `master`, `replica` or `standby-leader`.
- Experimental Windows support (Pavel Golub)
From now on it is possible to run Patroni on Windows, although Windows support is brand-new and hasn't received as much real-world testing as its Linux counterpart. We welcome your feedback!
**Improvements in patronictl**
- Add patronictl -k/--insecure flag and support for restapi cert (Wilfried Roset)
In the past if the REST API was protected by the self-signed certificates `patronictl` would fail to verify them. There was no way to disable that verification. It is now possible to configure `patronictl` to skip the certificate verification altogether or provide CA and client certificates in the :ref:`ctl: <patronictl_settings>` section of configuration.
- Exclude members with nofailover tag from patronictl switchover/failover output (Alexander Anikin)
Previously, those members were incorrectly proposed as candidates when performing interactive switchover or failover via patronictl.
**Stability improvements**
- Avoid parsing non-key-value output lines of pg_controldata (Alexander Anikin)
Under certain circuimstances pg_controldata outputs lines without a colon character. That would trigger an error in Patroni code that parsed pg_controldata output, hiding the actual problem; often such lines are emitted in a warning shown by pg_controldata before the regular output, i.e. when the binary major version does not match the one of the PostgreSQL data directory.
- Add member name to the error message during the leader election (Jan Mussler)
During the leader election, Patroni connects to all known members of the cluster and requests their status. Such status is written to the Patroni log and includes the name of the member. Previously, if the member was not accessible, the error message did not indicate its name, containing only the URL.
- Immediately reserve the WAL position upon creation of the replication slot (Alexander Kukushkin)
Starting from 9.6, `pg_create_physical_replication_slot` function provides an additional boolean parameter `immediately_reserve`. When it is set to `false`, which is also the default, the slot doesn't reserve the WAL position until it receives the first client connection, potentially losing some segments required by the client in a time window between the slot creation and the intiial client connection.
- Fix bug in strict synchronous replication (Alexander Kukushkin)
When running with `synchronous_mode_strict: true`, in some cases Patroni puts `*` into the `synchronous_standby_names`, changing the sync state for most of the replication connections to `potential`. Previously, Patroni couldn't pick a synchronous candidate under such curcuimstances, as it only considered those with the state `async`.
Version 1.4.6
-------------
**Bug fixes and stability improvements**
This release fixes a critical issue with Patroni API /master endpoint returning 200 for the non-master node. This is a
reporting issue, no actual split-brain, but under certain circumstances clients might be directed to the read-only node.
- Reset is_leader status on demote (Alexander Kukushkin, Oleksii Kliukin)
Make sure demoted cluster member stops responding with code 200 on the /master API call.
- Add new "cluster_unlocked" field to the API output (Dmitry Dolgov)
This field indicates whether the cluster has the master running. It can be used when it is not possible to query any
other node but one of the replicas.
Version 1.4.5
-------------
**New features**
- Improve logging when applying new postgres configuration (Don Seiler)
Patroni logs changed parameter names and values.
- Python 3.7 compatibility (Christoph Berg)
async is a reserved keyword in python3.7
- Set state to "stopped" in the DCS when a member is shut down (Tony Sorrentino)
This shows the member state as "stopped" in "patronictl list" command.
- Improve the message logged when stale postmaster.pid matches a running process (Ants Aasma)
The previous one was beyond confusing.
- Implement patronictl reload functionality (Don Seiler)
Before that it was only possible to reload configuration by either calling REST API or by sending SIGHUP signal to the Patroni process.
- Take and apply some parameters from controldata when starting as a replica (Alexander Kukushkin)
The value of `max_connections` and some other parameters set in the global configuration may be lower than the one actually used by the primary; when this happens, the replica cannot start and should be fixed manually. Patroni takes care of that now by reading and applying the value from `pg_controldata`, starting postgres and setting `pending_restart` flag.
- If set, use LD_LIBRARY_PATH when starting postgres (Chris Fraser)
When starting up Postgres, Patroni was passing along PATH, LC_ALL and LANG env vars if they are set. Now it is doing the same with LD_LIBRARY_PATH. It should help if somebody installed PostgreSQL to non-standard place.
- Rename create_replica_method to create_replica_methods (Dmitry Dolgov)
To make it clear that it's actually an array. The old name is still supported for backward compatibility.
**Bug fixes and stability improvements**
- Fix condition for the replica start due to pg_rewind in paused state (Oleksii Kliukin)
Avoid starting the replica that had already executed pg_rewind before.
- Respond 200 to the master health-check only if update_lock has been successful (Alexander)
Prevent Patroni from reporting itself a master on the former (demoted) master if DCS is partitioned.
- Fix compatibility with the new consul module (Alexander)
Starting from v1.1.0 python-consul changed internal API and started using `list` instead of `dict` to pass query parameters.
- Catch exceptions from Patroni REST API thread during shutdown (Alexander)
Those uncaught exceptions kept PostgreSQL running at shutdown.
- Do crash recovery only when Postgres runs as the master (Alexander)
Require `pg_controldata` to report 'in production' or 'shutting down' or 'in crash recovery'. In all other cases no crash recovery is necessary.
- Improve handling of configuration errors (Henning Jacobs, Alexander)
It is possible to change a lot of parameters in runtime (including `restapi.listen`) by updating Patroni config file and sending SIGHUP to Patroni process. This fix eliminates obscure exceptions from the 'restapi' thread when some of the parameters receive invalid values.
Version 1.4.4
-------------
**Stability improvements**
- Fix race condition in poll_failover_result (Alexander Kukushkin)
It didn't affect directly neither failover nor switchover, but in some rare cases it was reporting success too early, when the former leader released the lock, producing a 'Failed over to "None"' instead of 'Failed over to "desired-node"' message.
- Treat Postgres parameter names as case insensitive (Alexander)
Most of the Postgres parameters have snake_case names, but there are three exceptions from this rule: DateStyle, IntervalStyle and TimeZone. Postgres accepts those parameters when written in a different case (e.g. timezone = 'some/tzn'); however, Patroni was unable to find case-insensitive matches of those parameter names in pg_settings and ignored such parameters as a result.
- Abort start if attaching to running postgres and cluster not initialized (Alexander)
Patroni can attach itself to an already running Postgres instance. It is imperative to start running Patroni on the master node before getting to the replicas.
- Fix behavior of patronictl scaffold (Alexander)
Pass dict object to touch_member instead of json encoded string, DCS implementation will take care of encoding it.
- Don't demote master if failed to update leader key in pause (Alexander)
During maintenance a DCS may start failing write requests while continuing to responds to read ones. In that case, Patroni used to put the Postgres master node to a read-only mode after failing to update the leader lock in DCS.
- Sync replication slots when Patroni notices a new postmaster process (Alexander)
If Postgres has been restarted, Patroni has to make sure that list of replication slots matches its expectations.
- Verify sysid and sync replication slots after coming out of pause (Alexander)
During the `maintenance` mode it may happen that data directory was completely rewritten and therefore we have to make sure that `Database system identifier` still belongs to our cluster and replication slots are in sync with Patroni expectations.
- Fix a possible failure to start not running Postgres on a data directory with postmaster lock file present (Alexander)
Detect reuse of PID from the postmaster lock file. More likely to hit such problem if you run Patroni and Postgres in the docker container.
- Improve protection of DCS being accidentally wiped (Alexander)
Patroni has a lot of logic in place to prevent failover in such case; it can also restore all keys back; however, until this change an accidental removal of /config key was switching off pause mode for 1 cycle of HA loop.
- Do not exit when encountering invalid system ID (Oleksii Kliukin)
Do not exit when the cluster system ID is empty or the one that doesn't pass the validation check. In that case, the cluster most likely needs a reinit; mention it in the result message. Avoid terminating Patroni, as otherwise reinit cannot happen.
**Compatibility with Kubernetes 1.10+**
- Added check for empty subsets (Cody Coons)
Kubernetes 1.10.0+ started returning `Endpoints.subsets` set to `None` instead of `[]`.
**Bootstrap improvements**
- Make deleting recovery.conf optional (Brad Nicholson)
If `bootstrap.<custom_bootstrap_method_name>.keep_existing_recovery_conf` is defined and set to ``True``, Patroni will not remove the existing ``recovery.conf`` file. This is useful when bootstrapping from a backup with tools like pgBackRest that generate the appropriate `recovery.conf` for you.
- Allow options to the basebackup built-in method (Oleksii)
It is now possible to supply options to the built-in basebackup method by defining the `basebackup` section in the configuration, similar to how those are defined for custom replica creation methods. The difference is in the format accepted by the `basebackup` section: since pg_basebackup accepts both `--key=value` and `--key` options, the contents of the section could be either a dictionary of key-value pairs, or a list of either one-element dictionaries or just keys (for the options that don't accept values). See :ref:`replica creation method <custom_replica_creation>` section for additional examples.
Version 1.4.3
-------------
**Improvements in logging**
- Make log level configurable from environment variables (Andy Newton, Keyvan Hedayati)
`PATRONI_LOGLEVEL` - sets the general logging level
`PATRONI_REQUESTS_LOGLEVEL` - sets the logging level for all HTTP requests e.g. Kubernetes API calls
See `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>` to get the names of possible log levels
**Stability improvements and bug fixes**
- Don't rediscover etcd cluster topology when watch timed out (Alexander Kukushkin)
If we have only one host in etcd configuration and exactly this host is not accessible, Patroni was starting discovery of cluster topology and never succeeding. Instead it should just switch to the next available node.
- Write content of bootstrap.pg_hba into a pg_hba.conf after custom bootstrap (Alexander)
Now it behaves similarly to the usual bootstrap with `initdb`
- Single user mode was waiting for user input and never finish (Alexander)
Regression was introduced in https://github.com/zalando/patroni/pull/576
Version 1.4.2
-------------
**Improvements in patronictl**
- Rename scheduled failover to scheduled switchover (Alexander Kukushkin)
Failover and switchover functions were separated in version 1.4, but `patronictl list` was still reporting `Scheduled failover` instead of `Scheduled switchover`.
- Show information about pending restarts (Alexander)
In order to apply some configuration changes sometimes it is necessary to restart postgres. Patroni was already giving a hint about that in the REST API and when writing node status into DCS, but there were no easy way to display it.
- Make show-config to work with cluster_name from config file (Alexander)
It works similar to the `patronictl edit-config`
**Stability improvements**
- Avoid calling pg_controldata during bootstrap (Alexander)
During initdb or custom bootstrap there is a time window when pgdata is not empty but pg_controldata has not been written yet. In such case pg_controldata call was failing with error messages.
- Handle exceptions raised from psutil (Alexander)
cmdline is read and parsed every time when `cmdline()` method is called. It could happen that the process being examined
has already disappeared, in that case `NoSuchProcess` is raised.
**Kubernetes support improvements**
- Don't swallow errors from k8s API (Alexander)
A call to Kubernetes API could fail for a different number of reasons. In some cases such call should be retried, in some other cases we should log the error message and the exception stack trace. The change here will help debug Kubernetes permission issues.
- Update Kubernetes example Dockerfile to install Patroni from the master branch (Maciej Szulik)
Before that it was using `feature/k8s`, which became outdated.
- Add proper RBAC to run patroni on k8s (Maciej)
Add the Service account that is assigned to the pods of the cluster, the role that holds only the necessary permissions, and the rolebinding that connects the Service account and the Role.
Version 1.4.1
-------------
**Fixes in patronictl**
- Don't show current leader in suggested list of members to failover to. (Alexander Kukushkin)
patronictl failover could still work when there is leader in the cluster and it should be excluded from the list of member where it is possible to failover to.
- Make patronictl switchover compatible with the old Patroni api (Alexander)
In case if POST /switchover REST API call has failed with status code 501 it will do it once again, but to /failover endpoint.
Version 1.4
-----------
This version adds support for using Kubernetes as a DCS, allowing to run Patroni as a cloud-native agent in Kubernetes without any additional deployments of Etcd, Zookeeper or Consul.
**Upgrade notice**
Installing Patroni via pip will no longer bring in dependencies for (such as libraries for Etcd, Zookeper, Consul or Kubernetes, or support for AWS). In order to enable them one need to list them in pip install command explicitely, for instance `pip install patroni[etcd,kubernetes]`.
**Kubernetes support**
Implement Kubernetes-based DCS. The endpoints meta-data is used in order to store the configuration and the leader key. The meta-data field inside the pods definition is used to store the member-related data.
In addition to using Endpoints, Patroni supports ConfigMaps. You can find more information about this feature in the :ref:`Kubernetes chapter of the documentation <kubernetes>`
**Stability improvements**
- Factor out postmaster process into a separate object (Ants Aasma)
This object identifies a running postmaster process via pid and start time and simplifies detection (and resolution) of situations when the postmaster was restarted behind our back or when postgres directory disappeared from the file system.
- Minimize the amount of SELECT's issued by Patroni on every loop of HA cylce (Alexander Kukushkin)
On every iteration of HA loop Patroni needs to know recovery status and absolute wal position. From now on Patroni will run only single SELECT to get this information instead of two on the replica and three on the master.
- Remove leader key on shutdown only when we have the lock (Ants)
Unconditional removal was generating unnecessary and missleading exceptions.
**Improvements in patronictl**
- Add version command to patronictl (Ants)
It will show the version of installed Patroni and versions of running Patroni instances (if the cluster name is specified).
- Make optional specifying cluster_name argument for some of patronictl commands (Alexander, Ants)
It will work if patronictl is using usual Patroni configuration file with the ``scope`` defined.
- Show information about scheduled switchover and maintenance mode (Alexander)
Before that it was possible to get this information only from Patroni logs or directly from DCS.
- Improve ``patronictl reinit`` (Alexander)
Sometimes ``patronictl reinit`` refused to proceed when Patroni was busy with other actions, namely trying to start postgres. `patronictl` didn't provide any commands to cancel such long running actions and the only (dangerous) workarond was removing a data directory manually. The new implementation of `reinit` forcefully cancells other long-running actions before proceeding with reinit.
- Implement ``--wait`` flag in ``patronictl pause`` and ``patronictl resume`` (Alexander)
It will make ``patronictl`` wait until the requested action is acknowledged by all nodes in the cluster.
Such behaviour is achieved by exposing the ``pause`` flag for every node in DCS and via the REST API.
- Rename ``patronictl failover`` into ``patronictl switchover`` (Alexander)
The previous ``failover`` was actually only capable of doing a switchover; it refused to proceed in a cluster without the leader.
- Alter the behavior of ``patronictl failover`` (Alexander)
It will work even if there is no leader, but in that case you will have to explicitely specify a node which should become the new leader.
**Expose information about timeline and history**
- Expose current timeline in DCS and via API (Alexander)
Store information about the current timeline for each member of the cluster. This information is accessible via the API and is stored in the DCS
- Store promotion history in the /history key in DCS (Alexander)
In addition, store the timeline history enriched with the timestamp of the corresponding promotion in the /history key in DCS and update it with each promote.
**Add endpoints for getting synchronous and asynchronous replicas**
- Add new /sync and /async endpoints (Alexander, Oleksii Kliukin)
Those endpoints (also accessible as /synchronous and /asynchronous) return 200 only for synchronous and asynchornous replicas correspondingly (exclusing those marked as `noloadbalance`).
**Allow multiple hosts for Etcd**
- Add a new `hosts` parameter to Etcd configuration (Alexander)
This parameter should contain the initial list of hosts that will be used to discover and populate the list of the running etcd cluster members. If for some reason during work this list of discovered hosts is exhausted (no available hosts from that list), Patroni will return to the initial list from the `hosts` parameter.
Version 1.3.6
-------------
**Stability improvements**
- Verify process start time when checking if postgres is running. (Ants Aasma)
After a crash that doesn't clean up postmaster.pid there could be a new process with the same pid, resulting in a false positive for is_running(), which will lead to all kinds of bad behavior.
- Shutdown postgresql before bootstrap when we lost data directory (ainlolcat)
When data directory on the master is forcefully removed, postgres process can still stay alive for some time and prevent the replica created in place of that former master from starting or replicating.
The fix makes Patroni cache the postmaster pid and its start time and let it terminate the old postmaster in case it is still running after the corresponding data directory has been removed.
- Perform crash recovery in a single user mode if postgres master dies (Alexander Kukushkin)
It is unsafe to start immediately as a standby and not possible to run ``pg_rewind`` if postgres hasn't been shut down cleanly.
The single user crash recovery only kicks in if ``pg_rewind`` is enabled or there is no master at the moment.
**Consul improvements**
- Make it possible to provide datacenter configuration for Consul (Vilius Okockis, Alexander)
Before that Patroni was always communicating with datacenter of the host it runs on.
- Always send a token in X-Consul-Token http header (Alexander)
If ``consul.token`` is defined in Patroni configuration, we will always send it in the 'X-Consul-Token' http header.
python-consul module tries to be "consistent" with Consul REST API, which doesn't accept token as a query parameter for `session API <https://www.consul.io/api/session.html>`__, but it still works with 'X-Consul-Token' header.
- Adjust session TTL if supplied value is smaller than the minimum possible (Stas Fomin, Alexander)
It could happen that the TTL provided in the Patroni configuration is smaller than the minimum one supported by Consul. In that case, Consul agent fails to create a new session.
Without a session Patroni cannot create member and leader keys in the Consul KV store, resulting in an unhealthy cluster.
**Other improvements**
- Define custom log format via environment variable ``PATRONI_LOGFORMAT`` (Stas)
Allow disabling timestamps and other similar fields in Patroni logs if they are already added by the system logger (usually when Patroni runs as a service).
Version 1.3.5
-------------
**Bugfix**
- Set role to 'uninitialized' if data directory was removed (Alexander Kukushkin)
If the node was running as a master it was preventing from failover.
**Stability improvement**
- Try to run postmaster in a single-user mode if we tried and failed to start postgres (Alexander)
Usually such problem happens when node running as a master was terminated and timelines were diverged.
If ``recovery.conf`` has ``restore_command`` defined, there are really high chances that postgres will abort startup and leave controldata unchanged.
It makes impossible to use ``pg_rewind``, which requires a clean shutdown.
**Consul improvements**
- Make it possible to specify health checks when creating session (Alexander)
If not specified, Consul will use "serfHealth". From one side it allows fast detection of isolated master, but from another side it makes it impossible for Patroni to tolerate short network lags.
**Bugfix**
- Fix watchdog on Python 3 (Ants Aasma)
A misunderstanding of the ioctl() call interface. If mutable=False then fcntl.ioctl() actually returns the arg buffer back.
This accidentally worked on Python2 because int and str comparison did not return an error.
Error reporting is actually done by raising IOError on Python2 and OSError on Python3.
Version 1.3.4
-------------
**Different Consul improvements**
- Pass the consul token as a header (Andrew Colin Kissa)
Headers are now the prefered way to pass the token to the consul `API <https://www.consul.io/api/index.html#authentication>`__.
- Advanced configuration for Consul (Alexander Kukushkin)
possibility to specify ``scheme``, ``token``, client and ca certificates :ref:`details <consul_settings>`.
- compatibility with python-consul-0.7.1 and above (Alexander)
new python-consul module has changed signature of some methods
- "Could not take out TTL lock" message was never logged (Alexander)
Not a critical bug, but lack of proper logging complicates investigation in case of problems.
**Quote synchronous_standby_names using quote_ident**
- When writing ``synchronous_standby_names`` into the ``postgresql.conf`` its value must be quoted (Alexander)
If it is not quoted properly, PostgreSQL will effectively disable synchronous replication and continue to work.
**Different bugfixes around pause state, mostly related to watchdog** (Alexander)
- Do not send keepalives if watchdog is not active
- Avoid activating watchdog in a pause mode
- Set correct postgres state in pause mode
- Do not try to run queries from API if postgres is stopped
Version 1.3.3
-------------
**Bugfixes**
- synchronous replication was disabled shortly after promotion even when synchronous_mode_strict was turned on (Alexander Kukushkin)
- create empty ``pg_ident.conf`` file if it is missing after restoring from the backup (Alexander)
- open access in ``pg_hba.conf`` to all databases, not only postgres (Franco Bellagamba)
Version 1.3.2
-------------
**Bugfix**
- patronictl edit-config didn't work with ZooKeeper (Alexander Kukushkin)
Version 1.3.1
-------------
**Bugfix**
- failover via API was broken due to change in ``_MemberStatus`` (Alexander Kukushkin)
Version 1.3
-----------
+7 -116
View File
@@ -3,7 +3,7 @@ Replica imaging and bootstrap
Patroni allows customizing creation of a new replica. It also supports defining what happens when the new empty cluster
is being bootstrapped. The distinction between two is well defined: Patroni creates replicas only if the ``initialize``
key is present in DCS for the cluster. If there is no ``initialize`` key - Patroni calls bootstrap exclusively on the
key is present in Etcd for the cluster. If there is no ``initialize`` key - Patroni calls bootstrap exclusively on the
first node that takes the initialize key lock.
.. _custom_bootstrap:
@@ -23,7 +23,6 @@ arguments to them, i.e. the name of the cluster and the path to the data directo
method: <custom_bootstrap_method_name>
<custom_bootstrap_method_name>:
command: <path_to_custom_bootstrap_script> [param1 [, ...]]
keep_existing_recovery_conf: False
recovery_conf:
recovery_target_action: promote
recovery_target_timeline: latest
@@ -48,9 +47,6 @@ If a ``recovery_conf`` block is defined in the same section as the custom bootst
``recovery.conf`` before starting the newly bootstrapped instance. Typically, such recovery.conf should contain at least
one of the ``recovery_target_*`` parameters, together with the ``recovery_target_timeline`` set to ``promote``.
If ``keep_existing_recovery_conf`` is defined and set to ``True``, Patroni will not remove the existing ``recovery.conf`` file if it exists.
This is useful when bootstrapping from a backup with tools like pgBackRest that generate the appropriate ``recovery.conf`` for you.
.. note:: Bootstrap methods are neither chained, nor fallen-back to the default one in case the primary one fails
@@ -68,20 +64,7 @@ scripts to clone a new replica. Those are configured in the ``postgresql`` confi
.. code:: YAML
postgresql:
create_replica_methods:
- <method name>
<method name>:
command: <command name>
keep_data: True
no_params: True
no_master: 1
example: wal_e
.. code:: YAML
postgresql:
create_replica_methods:
create_replica_method:
- wal_e
- basebackup
wal_e:
@@ -89,29 +72,13 @@ example: wal_e
no_master: 1
envdir: {{WALE_ENV_DIR}}
use_iam: 1
basebackup:
max-rate: '100M'
example: pgbackrest
.. code:: YAML
postgresql:
create_replica_methods:
- pgbackrest
- basebackup
pgbackrest:
command: /usr/bin/pgbackrest --stanza=<scope> --delta restore
keep_data: True
no_params: True
basebackup:
max-rate: '100M'
The ``create_replica_methods`` defines available replica creation methods and the order of executing them. Patroni will
stop on the first one that returns 0. Each method should define a separate section in the configuration file, listing the command
to execute and any custom parameters that should be passed to that command. All parameters will be passed in a
``--name=value`` format. Besides user-defined parameters, Patroni supplies a couple of cluster-specific ones:
The ``create_replica_method`` defines available replica creation methods and the order of executing them. Patroni will
stop on the first one that returns 0. The basebackup is the built-in method and doesn't require any configuration. The
rest of the methods should define a separate section in the configuration file, listing the command to execute and any
custom parameters that should be passed to that command. All parameters will be passed in a ``--name=value`` format.
Besides user-defined parameters, Patroni supplies a couple of cluster-specific ones:
--scope
Which cluster this replica belongs to
@@ -127,81 +94,5 @@ A special ``no_master`` parameter, if defined, allows Patroni to call the replic
running master or replicas. In that case, an empty string will be passed in a connection string. This is useful for
restoring the formerly running cluster from the binary backup.
A special ``keep_data`` parameter, if defined, will instuct Patroni to not clean PGDATA folder before calling restore.
A special ``no_params`` parameter, if defined, restricts passing parameters to custom command.
A ``basebackup`` method is a special case: it will be used if
``create_replica_methods`` is empty, although it is possible
to list it explicitly among the ``create_replica_methods`` methods. This method initializes a new replica with the
``pg_basebackup``, the base backup is taken from the master unless there are replicas with ``clonefrom`` tag, in which case one
of such replicas will be used as the origin for pg_basebackup. It works without any configuration; however, it is
possible to specify a ``basebackup`` configuration section. Same rules as with the other method configuration apply,
namely, only long (with --) options should be specified there. Not all parameters make sense, if you override a connection
string or provide an option to created tar-ed or compressed base backups, patroni won't be able to make a replica out
of it. There is no validation performed on the names or values of the parameters passed to the ``basebackup`` section.
You can specify basebackup parameters as either a map (key-value pairs) or a list of elements, where each element
could be either a key-value pair or a single key (for options that does not receive any values, for instance, ``--verbose``).
Consider those 2 examples:
.. code:: YAML
postgresql:
basebackup:
max-rate: '100M'
checkpoint: 'fast'
and
.. code:: YAML
postgresql:
basebackup:
- verbose
- max-rate: '100M'
If all replica creation methods fail, Patroni will try again all methods in order during the next event loop cycle.
.. _standby_cluster:
Standby cluster
---------------
Another available option is to run a "standby cluster", that contains only of
standby nodes replicating from some remote master. This type of clusters has:
* "standby leader", that behaves pretty much like a regular cluster leader,
except it replicates from a remote master.
* cascade replicas, that are replicating from standby leader.
Standby leader holds and updates a leader lock in DCS. If the leader lock
expires, cascade replicas will perform an election to choose another leader
from the standbys.
For the sake of flexibility, you can specify methods of creating a replica and
recovery WAL records when a cluster is in the "standby mode" by providing
`create_replica_methods` key in `standby_cluster` section. It is distinct from
creating replicas, when cluster is detached and functions as a normal cluster,
which is controlled by `create_replica_methods` in `postgresql` section. Both
"standby" and "normal" `create_replica_methods` reference keys in `postgresql`
section.
To configure such cluster you need to specify the section ``standby_cluster``
in a patroni configuration:
.. code:: YAML
bootstrap:
dcs:
standby_cluster:
host: 1.2.3.4
port: 5432
primary_slot_name: patroni
create_replica_methods:
- basebackup
Note, that these options will be applied only once during cluster bootstrap,
and the only way to change them afterwards is through DCS.
If you use replication slots on the standby cluster, you must also create the corresponding replication slot on the primary cluster. It will not be done automatically by the standby cluster implementation. You can use Patroni's permenant replication slots feature on the primary cluster to maintain a replication slot with the same name as ``primary_slot_name``, or its default value if ``primary_slot_name`` is not provided.
+11 -22
View File
@@ -9,11 +9,9 @@ Patroni uses PostgreSQL streaming replication. For more information about stream
Asynchronous mode durability
----------------------------
In asynchronous mode the cluster is allowed to lose some committed transactions to ensure availability. When the primary server fails or becomes unavailable for any other reason Patroni will automatically promote a sufficiently healthy standby to primary. Any transactions that have not been replicated to that standby remain in a "forked timeline" on the primary, and are effectively unrecoverable [1]_.
In asynchronous mode the cluster is allowed to lose some committed transactions to ensure availability. When master server fails or becomes unavailable for any other reason Patroni will automatically promote a sufficiently healthy standby to master. Any transactions that have not been replicated to that standby remain in a "forked timeline" on the master, and are effectively unrecoverable [1]_.
The amount of transactions that can be lost is controlled via ``maximum_lag_on_failover`` parameter. Because the primary transaction log position is not sampled in real time, in reality the amount of lost data on failover is worst case bounded by ``maximum_lag_on_failover`` bytes of transaction log plus the amount that is written in the last ``ttl`` seconds (``loop_wait``/2 seconds in the average case). However typical steady state replication delay is well under a second.
By default, when running leader elections, Patroni does not take into account the current timeline of replicas, what in some cases could be undesirable behavior. You can prevent the node not having the same timeline as a former master become the new leader by changing the value of ``check_timeline`` parameter to ``true``.
The amount of transactions that can be lost is controlled via ``maximum_lag_on_failover`` parameter. Because master transaction log position is not sampled in real time, in reality the amount of lost data on failover is worst case bounded by ``maximum_lag_on_failover`` bytes of transaction log plus the amount that is written in the last ``ttl`` seconds (``loop_wait``/2 seconds in the average case). However typical steady state replication delay is well under a second.
PostgreSQL synchronous replication
----------------------------------
@@ -22,7 +20,7 @@ You can use Postgres's `synchronous replication <http://www.postgresql.org/docs/
In hosted datacenter environments (like AWS, Rackspace, or any network you do not control), synchronous replication significantly increases the variability of write performance. If followers become inaccessible from the leader, the leader effectively becomes read-only.
To enable a simple synchronous replication test, add the following lines to the ``parameters`` section of your YAML configuration files:
To enable a simple synchronous replication test, add the follow lines to the ``parameters`` section of your YAML configuration files:
.. code:: YAML
@@ -31,27 +29,18 @@ To enable a simple synchronous replication test, add the following lines to the
When using PostgreSQL synchronous replication, use at least three Postgres data nodes to ensure write availability if one host fails.
Using PostgreSQL synchronous replication does not guarantee zero lost transactions under all circumstances. When the primary and the secondary that is currently acting as a synchronous replica fail simultaneously a third node that might not contain all transactions will be promoted.
Using PostgreSQL synchronous replication does not guarantee zero lost transactions under all circumstances. When master and standby that is currently acting as synchronous fail simultaneously a third node that might not contain all transactions will be promoted.
.. _synchronous_mode:
Synchronous mode
----------------
For use cases where losing committed transactions is not permissible you can turn on Patroni's ``synchronous_mode``. When ``synchronous_mode`` is turned on Patroni will not promote a standby unless it is certain that the standby contains all transactions that may have returned a successful commit status to client [2]_. This means that the system may be unavailable for writes even though some servers are available. System administrators can still use manual failover commands to promote a standby even if it results in transaction loss.
For use cases where losing committed transactions is not permissible you can turn on Patronis ``synchronous_mode``. When ``synchronous_mode`` is turned on Patroni will not promote a standby unless it is certain that the standby contains all transactions that may have returned a successful commit status to client [2]_. This means that the system may be unavailable for writes even though some servers are available. System administrators can still use manual failover commmands to promote a standby even if it results in transaction loss.
Turning on ``synchronous_mode`` does not guarantee multi node durability of commits under all circumstances. When no suitable standby is available, primary server will still accept writes, but does not guarantee their replication. When the primary fails in this mode no standby will be promoted. When the host that used to be the primary comes back it will get promoted automatically, unless system administrator performed a manual failover. This behavior makes synchronous mode usable with 2 node clusters.
Turning on ``synchronous_mode`` does not guarantee multi node durability of commits under all circumstances. When no suitable standby is available, master server will still accept writes, but does not guarantee their replication. When the master fails in this mode no standby will be promote. When the host that used to be master comes back it will get promoted automatically, unless system administrator performed a manual failover. This behavior makes synchronous mode usable with 2 node clusters.
When ``synchronous_mode`` is on and a standby crashes, commits will block until next iteration of Patroni runs and switches the primary to standalone mode (worst case delay for writes ``ttl`` seconds, average case ``loop_wait``/2 seconds). Manually shutting down or restarting a standby will not cause a commit service interruption. Standby will signal the primary to release itself from synchronous standby duties before PostgreSQL shutdown is initiated.
When it is absolutely necessary to guarantee that each write is stored durably
on at least two nodes, enable ``synchronous_mode_strict`` in addition to the
``synchronous_node``. This parameter prevents Patroni from switching off the
synchronous replication on the primary when no synchronous standby candidates
are available. As a downside, the primary is not be available for writes
(unless the Postgres transaction explicitly turns of ``synchronous_mode``),
blocking all client write requests until at least one synchronous replica comes
up.
When ``synchronous_mode`` is on and a standby crashes, commits will block until next iteration of Patroni runs and switches master to standalone mode (worst case delay for writes ``ttl`` seconds, average case ``loop_wait``/2 seconds). Manually shutting down or restarting a standby will not cause a commit service interruption. Standby will signal the master to release itself from synchronous standby duties before PostgreSQL shutdown is initiated.
You can ensure that a standby never becomes the synchronous standby by setting ``nosync`` tag to true. This is recommended to set for standbys that are behind slow network connections and would cause performance degradation when becoming a synchronous standby.
@@ -61,7 +50,7 @@ Synchronous mode can be switched on and off via Patroni REST interface. See :ref
Synchronous mode implementation
-------------------------------
When in synchronous mode Patroni maintains synchronization state in the DCS, containing the latest primary and current synchronous standby. This state is updated with strict ordering constraints to ensure the following invariants:
When in synchronous mode Patroni maintains synchronization state in the DCS, containing the latest master and current synchronous standby. This state is updated with strict ordering constraints to ensure the following invariants:
- A node must be marked as the latest leader whenever it can accept write transactions. Patroni crashing or PostgreSQL not shutting down can cause violations of this invariant.
@@ -71,9 +60,9 @@ When in synchronous mode Patroni maintains synchronization state in the DCS, con
Patroni will only ever assign one standby to ``synchronous_standby_names`` because with multiple candidates it is not possible to know which node was acting as synchronous during the failure.
On each HA loop iteration Patroni re-evaluates synchronous standby choice. If the current synchronous standby is connected and has not requested its synchronous status to be removed it remains picked. Otherwise the cluster member available for sync that is furthest ahead in replication is picked.
On each HA loop iteration Patroni re-evaluates synchronous standby choice. If the current synchronous standby is connected and has not requested its synchronous status to be removed it remains picked. Otherwise the cluster member avaiable for sync that is furthest ahead in replication is picked.
.. [1] The data is still there, but recovering it requires a manual recovery effort by data recovery specialists. When Patroni is allowed to rewind with ``use_pg_rewind`` the forked timeline will be automatically erased to rejoin the failed primary with the cluster.
.. [1] The data is still there, but recovering it requires a manual recovery effort by data recovery specialists. When Patroni is allowed to rewind with ``use_pg_rewind`` the forked timeline will be automatically erased to rejoin the failed master with the cluster.
.. [2] Clients can change the behavior per transaction using PostgreSQL's ``synchronous_commit`` setting. Transactions with ``synchronous_commit`` values of ``off`` and ``local`` may be lost on fail over, but will not be blocked by replication delays.
.. [2] Clients can change the behavior per transaction using PostgreSQL's ``synchronous_commit`` setting. Transactions with ``synchronous_commit`` values of ``off`` and ``local`` may be lost on fail over, but will not be blocked by replication delays.
+2 -1
View File
@@ -1,5 +1,6 @@
.. _watchdog:
================
Watchdog support
================
@@ -9,7 +10,7 @@ Having multiple PostgreSQL servers running as master can result in transactions
- Shutting down PostgreSQL is too slow.
- Patroni does not get to run due to high load on the system, the VM being paused by the hypervisor, or other infrastructure issues.
- Patroni does not get to run due to high load on the system, th VM being paused by the hypervisor, or other infrastructure issues.
To guarantee correct behavior under these conditions Patroni supports watchdog devices. Watchdog devices are software or hardware mechanisms that will reset the whole system when they do not get a keepalive heartbeat within a specified timeframe. This adds an additional layer of fail safe in case usual Patroni split-brain protection mechanisms fail.
+3 -3
View File
@@ -1,11 +1,11 @@
### confd
`confd` directory contains haproxy and pgbouncer template files for the [confd](https://github.com/kelseyhightower/confd) -- lightweight configuration management tool
`confd` directory contains haproxy template files for the [confd](https://github.com/kelseyhightower/confd) -- lightweight configuration management tool
You need to copy content of `confd` directory into /etcd/confd and run confd service:
```bash
$ confd -prefix=/service/$PATRONI_SCOPE -backend etcd -node $PATRONI_ETCD_URL -interval=10
$ confd -prefix=/service/$PATRONI_SCOPE -backend etcd -node $PATRONI_ETCD_HOST -interval=10
```
It will periodically update haproxy.cfg and pgbouncer.ini with the actual list of Patroni nodes from `etcd` and "reload" haproxy and pgbouncer.ini when it is necessary.
It will periodically update haproxy.cfg with the actual list of Patroni nodes from `etcd` and "reload" haproxy when it is necessary.
### startup-scripts
-12
View File
@@ -1,12 +0,0 @@
[template]
prefix = "/service/batman"
owner = "postgres"
mode = "0644"
src = "pgbouncer.tmpl"
dest = "/etc/pgbouncer/pgbouncer.ini"
reload_cmd = "systemctl reload pgbouncer"
keys = [
"/members/","/leader"
]
+11 -15
View File
@@ -10,23 +10,19 @@ defaults
timeout server 30m
timeout check 5s
listen stats
mode http
bind *:7000
stats enable
stats uri /
listen master
frontend master_postgresql
bind *:5000
option httpchk OPTIONS /master
http-check expect status 200
default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
{{range gets "/members/*"}} server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}
{{end}}
listen replicas
default_backend backend_master
frontend replicas_postgresql
bind *:5001
default_backend backend_replicas
backend backend_master
option httpchk OPTIONS /master
{{range gets "/members/*"}} server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}
{{end}}
backend backend_replicas
option httpchk OPTIONS /replica
http-check expect status 200
default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
{{range gets "/members/*"}} server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}
{{end}}
-17
View File
@@ -1,17 +0,0 @@
[databases]
{{with get "/leader"}}{{$leader := .Value}}{{$leadkey := printf "/members/%s" $leader}}{{with get $leadkey}}{{$data := json .Value}}{{$hostport := base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}}{{ $host := base (index (split $hostport ":") 0)}}{{ $port := base (index (split $hostport ":") 1)}}* = host={{ $host }} port={{ $port }} pool_size=10{{end}}{{end}}
[pgbouncer]
logfile = /var/log/postgresql/pgbouncer.log
pidfile = /var/run/postgresql/pgbouncer.pid
listen_addr = *
listen_port = 6432
unix_socket_dir = /var/run/postgresql
auth_type = trust
auth_file = /etc/pgbouncer/userlist.txt
auth_hba_file = /etc/pgbouncer/pg_hba.txt
admin_users = pgbouncer
stats_users = pgbouncer
pool_mode = session
max_client_conn = 100
default_pool_size = 20
+144 -144
View File
@@ -1,144 +1,144 @@
#!/bin/sh
#
### BEGIN INIT INFO
# Provides: patroni
# Required-Start: $remote_fs $syslog
# Required-Stop: $remote_fs $syslog
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: Patroni init script
# Description: Runners to orchestrate a high-availability PostgreSQL
### END INIT INFO
### BEGIN USER CONFIGURATION
CONF="/etc/patroni/postgres.yml"
LOGFILE="/var/log/patroni.log"
USER="postgres"
GROUP="postgres"
NAME=patroni
PATRONI="/opt/patroni/$NAME.py"
PIDFILE="/var/run/$NAME.pid"
# Set this parameter, if you have several Postgres versions installed
# POSTGRES_VERSION="9.4"
POSTGRES_VERSION=""
### END USER CONFIGURATION
. /lib/lsb/init-functions
# Loading this library for get_versions() function
if test ! -e /usr/share/postgresql-common/init.d-functions; then
log_failure_msg "Probably postgresql-common does not installed."
exit 1
else
. /usr/share/postgresql-common/init.d-functions
fi
# Is there Patroni executable?
if test ! -e $PATRONI; then
log_failure_msg "Patroni executable $PATRONI does not exist."
exit 1
fi
# Is there Patroni configuration file?
if test ! -e $CONF; then
log_failure_msg "Patroni configuration file $CONF does not exist."
exit 1
fi
# Create logfile if doesn't exist
if test ! -e $LOGFILE; then
log_action_msg "Creating logfile for Patroni..."
touch $LOGFILE
chown $USER:$GROUP $LOGFILE
fi
prepare_pgpath() {
if [ "$POSTGRES_VERSION" != "" ]; then
if [ -x /usr/lib/postgresql/$POSTGRES_VERSION/bin/pg_ctl ]; then
PGPATH="/usr/lib/postgresql/$POSTGRES_VERSION/bin"
else
log_failure_msg "Postgres version incorrect, check POSTGRES_VERSION variable."
exit 0
fi
else
get_versions
if echo $versions | grep -q -e "\s"; then
log_warning_msg "You have several Postgres versions installed. Please, use POSTGRES_VERSION to define correct environment."
else
versions=`echo $versions | sed -e 's/^[ \t]*//'`
PGPATH="/usr/lib/postgresql/$versions/bin"
fi
fi
}
get_pid() {
if test -e $PIDFILE; then
PID=`cat $PIDFILE`
CHILDPID=`ps --ppid $PID -o %p --no-headers`
else
log_failure_msg "Could not find PID file. Patroni probably down."
exit 1
fi
}
case "$1" in
start)
prepare_pgpath
PGPATH=$PATH:$PGPATH
log_success_msg "Starting Patroni\n"
exec start-stop-daemon --start --quiet \
--background \
--pidfile $PIDFILE --make-pidfile \
--chuid $USER:$GROUP \
--chdir `eval echo ~$USER` \
--exec $PATRONI \
--startas /bin/sh -- \
-c "/usr/bin/env PATH=$PGPATH /usr/bin/python $PATRONI $CONF >> $LOGFILE 2>&1"
;;
stop)
log_success_msg "Stopping Patroni"
get_pid
start-stop-daemon --stop --pid $CHILDPID
start-stop-daemon --stop --pidfile $PIDFILE --remove-pidfile --quiet
;;
reload)
log_success_msg "Reloading Patroni configuration"
get_pid
kill -HUP $CHILDPID
;;
status)
get_pid
if start-stop-daemon -T --pid $CHILDPID; then
log_success_msg "Patroni is running\n"
exit 0
else
log_warning_msg "Patroni in not running\n"
fi
;;
restart)
$0 stop
$0 start
;;
*)
echo "Usage: /etc/init.d/$NAME {start|stop|restart|reload|status}"
exit 1
;;
esac
if [ $? -eq 0 ]; then
echo .
exit 0
else
echo " failed"
exit 1
fi
#!/bin/sh
#
### BEGIN INIT INFO
# Provides: patroni
# Required-Start: $remote_fs $syslog
# Required-Stop: $remote_fs $syslog
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: Patroni init script
# Description: Runners to orchestrate a high-availability PostgreSQL
### END INIT INFO
### BEGIN USER CONFIGURATION
CONF="/etc/patroni/postgres.yml"
LOGFILE="/var/log/patroni.log"
USER="postgres"
GROUP="postgres"
NAME=patroni
PATRONI="/opt/patroni/$NAME.py"
PIDFILE="/var/run/$NAME.pid"
# Set this parameter, if you have several Postgres versions installed
# POSTGRES_VERSION="9.4"
POSTGRES_VERSION=""
### END USER CONFIGURATION
. /lib/lsb/init-functions
# Loading this library for get_versions() function
if test ! -e /usr/share/postgresql-common/init.d-functions; then
log_failure_msg "Probably postgresql-common does not installed."
exit 1
else
. /usr/share/postgresql-common/init.d-functions
fi
# Is there Patroni executable?
if test ! -e $PATRONI; then
log_failure_msg "Patroni executable $PATRONI does not exist."
exit 1
fi
# Is there Patroni configuration file?
if test ! -e $CONF; then
log_failure_msg "Patroni configuration file $CONF does not exist."
exit 1
fi
# Create logfile if doesn't exist
if test ! -e $LOGFILE; then
log_action_msg "Creating logfile for Patroni..."
touch $LOGFILE
chown $USER:$GROUP $LOGFILE
fi
prepare_pgpath() {
if [ "$POSTGRES_VERSION" != "" ]; then
if [ -x /usr/lib/postgresql/$POSTGRES_VERSION/bin/pg_ctl ]; then
PGPATH="/usr/lib/postgresql/$POSTGRES_VERSION/bin"
else
log_failure_msg "Postgres version incorrect, check POSTGRES_VERSION variable."
exit 0
fi
else
get_versions
if echo $versions | grep -q -e "\s"; then
log_warning_msg "You have several Postgres versions installed. Please, use POSTGRES_VERSION to define correct environment."
else
versions=`echo $versions | sed -e 's/^[ \t]*//'`
PGPATH="/usr/lib/postgresql/$versions/bin"
fi
fi
}
get_pid() {
if test -e $PIDFILE; then
PID=`cat $PIDFILE`
CHILDPID=`ps --ppid $PID -o %p --no-headers`
else
log_failure_msg "Could not find PID file. Patroni probably down."
exit 1
fi
}
case "$1" in
start)
prepare_pgpath
PGPATH=$PATH:$PGPATH
log_success_msg "Starting Patroni\n"
exec start-stop-daemon --start --quiet \
--background \
--pidfile $PIDFILE --make-pidfile \
--chuid $USER:$GROUP \
--chdir `eval echo ~$USER` \
--exec $PATRONI \
--startas /bin/sh -- \
-c "/usr/bin/env PATH=$PGPATH /usr/bin/python $PATRONI $CONF >> $LOGFILE 2>&1"
;;
stop)
log_success_msg "Stopping Patroni"
get_pid
start-stop-daemon --stop --pid $CHILDPID
start-stop-daemon --stop --pidfile $PIDFILE --remove-pidfile --quiet
;;
reload)
log_success_msg "Reloading Patroni configuration"
get_pid
kill -HUP $CHILDPID
;;
status)
get_pid
if start-stop-daemon -T --pid $CHILDPID; then
log_success_msg "Patroni is running\n"
exit 0
else
log_warning_msg "Patroni in not running\n"
fi
;;
restart)
$0 stop
$0 start
;;
*)
echo "Usage: /etc/init.d/$NAME {start|stop|restart|reload|status}"
exit 1
;;
esac
if [ $? -eq 0 ]; then
echo .
exit 0
else
echo " failed"
exit 1
fi
+1 -15
View File
@@ -11,26 +11,12 @@ Type=simple
User=postgres
Group=postgres
# Read in configuration file if it exists, otherwise proceed
EnvironmentFile=-/etc/patroni_env.conf
WorkingDirectory=~
# Where to send early-startup messages from the server
# This is normally controlled by the global default set by systemd
#StandardOutput=syslog
# StandardOutput=syslog
# Pre-commands to start watchdog device
# Uncomment if watchdog is part of your patroni setup
#ExecStartPre=-/usr/bin/sudo /sbin/modprobe softdog
#ExecStartPre=-/usr/bin/sudo /bin/chown postgres /dev/watchdog
# Start the patroni process
ExecStart=/bin/patroni /etc/patroni.yml
# Send HUP to reload from patroni.yml
ExecReload=/bin/kill -s HUP $MAINPID
# only kill the patroni process, not it's children, so it will gracefully stop postgres
KillMode=process
+8 -20
View File
@@ -14,28 +14,16 @@ Feature: basic replication
Then table foo is present on postgres2 after 20 seconds
Scenario: check restart of sync replica
Given I shut down postgres2
Then "sync" key in DCS has sync_standby=postgres1 after 5 seconds
When I start postgres2
And I shut down postgres1
Then "sync" key in DCS has sync_standby=postgres2 after 10 seconds
When I start postgres1
And "members/postgres1" key in DCS has state=running after 10 seconds
And I sleep for 2 seconds
When I issue a GET request to http://127.0.0.1:8010/sync
Then I receive a response code 200
When I issue a GET request to http://127.0.0.1:8009/async
Then I receive a response code 200
Given I run patronictl.py restart batman postgres2 --force
And "sync" key in DCS has sync_standby=postgres1 after 2 seconds
And I run patronictl.py restart batman postgres1 --force
Then I receive a response returncode 0
And "sync" key in DCS has sync_standby=postgres2 after 10 seconds
Scenario: check the basic failover in synchronous mode
Given I run patronictl.py pause batman
Then I receive a response returncode 0
When I sleep for 2 seconds
And I shut down postgres0
And I run patronictl.py resume batman
Then I receive a response returncode 0
And postgres2 role is the primary after 24 seconds
When I issue a PATCH request to http://127.0.0.1:8010/config with {"synchronous_mode": null, "master_start_timeout": 0}
When I kill postgres0
Then postgres2 role is the primary after 22 seconds
When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": null, "master_start_timeout": 0}
Then I receive a response code 200
When I add the table bar to postgres2
Then table bar is present on postgres1 after 20 seconds
-5
View File
@@ -1,5 +0,0 @@
#!/bin/bash
[[ "$3" == "master" ]] || exit
PGPASSWORD=zalando psql -h localhost -U postgres -p $1 -w -tAc "SELECT slot_name FROM pg_replication_slots WHERE slot_type = 'logical'" >> data/postgres0/label
+28 -114
View File
@@ -7,12 +7,10 @@ import kazoo.exceptions
import os
import psutil
import psycopg2
import json
import shutil
import signal
import six
import subprocess
import sys
import tempfile
import threading
import time
@@ -85,7 +83,7 @@ class AbstractController(object):
class PatroniController(AbstractController):
__PORT = 5360
__PORT = 5440
PATRONI_CONFIG = '{}.yml'
""" starts and stops individual patronis"""
@@ -100,7 +98,6 @@ class PatroniController(AbstractController):
else:
self.watchdog = None
self._scope = (custom_config or {}).get('scope', 'batman')
self._config = self._make_patroni_test_config(name, custom_config)
self._closables = []
@@ -118,41 +115,23 @@ class PatroniController(AbstractController):
except IOError:
return None
@staticmethod
def recursive_update(dst, src):
for k, v in src.items():
if k in dst and isinstance(dst[k], dict):
PatroniController.recursive_update(dst[k], v)
else:
dst[k] = v
def update_config(self, custom_config):
def add_tag_to_config(self, tag, value):
with open(self._config) as r:
config = yaml.safe_load(r)
self.recursive_update(config, custom_config)
config['tags']['tag'] = value
with open(self._config, 'w') as w:
yaml.safe_dump(config, w, default_flow_style=False)
self._scope = config.get('scope', 'batman')
def add_tag_to_config(self, tag, value):
self.update_config({'tags': {tag: value}})
def _start(self):
if self.watchdog:
self.watchdog.start()
if isinstance(self._context.dcs_ctl, KubernetesController):
self._context.dcs_ctl.create_pod(self._name[8:], self._scope)
os.environ['PATRONI_KUBERNETES_POD_IP'] = '10.0.0.' + self._name[-1]
return subprocess.Popen([sys.executable, '-m', 'coverage', 'run',
'--source=patroni', '-p', 'patroni.py', self._config],
return subprocess.Popen(['coverage', 'run', '--source=patroni', '-p', 'patroni.py', self._config],
stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory)
def stop(self, kill=False, timeout=15, postgres=False):
if postgres:
return subprocess.call(['pg_ctl', '-D', self._data_dir, 'stop', '-mi', '-w'])
super(PatroniController, self).stop(kill, timeout)
if isinstance(self._context.dcs_ctl, KubernetesController):
self._context.dcs_ctl.delete_pod(self._name[8:])
if self.watchdog:
self.watchdog.stop()
@@ -168,7 +147,7 @@ class PatroniController(AbstractController):
with open(patroni_config_name) as f:
config = yaml.safe_load(f)
config.pop('etcd', None)
config.pop('etcd')
host = config['postgresql']['listen'].split(':')[0]
@@ -188,10 +167,13 @@ class PatroniController(AbstractController):
config['bootstrap']['initdb'].extend([{'auth': 'md5'}, {'auth-host': 'md5'}])
if custom_config is not None:
self.recursive_update(config, custom_config)
if config['postgresql'].get('callbacks', {}).get('on_role_change'):
config['postgresql']['callbacks']['on_role_change'] += ' ' + str(self.__PORT)
def recursive_update(dst, src):
for k, v in src.items():
if k in dst and isinstance(dst[k], dict):
recursive_update(dst[k], v)
else:
dst[k] = v
recursive_update(config, custom_config)
with open(patroni_config_path, 'w') as f:
yaml.safe_dump(config, f, default_flow_style=False)
@@ -246,7 +228,7 @@ class PatroniController(AbstractController):
if not os.path.exists(pidfile):
return None
return int(open(pidfile).readline().strip())
except Exception:
except:
return None
def database_is_running(self):
@@ -353,6 +335,10 @@ class AbstractDcsController(AbstractController):
def query(self, key, scope='batman'):
""" query for a value of a given key """
@abc.abstractmethod
def set(self, key, value):
""" set a value to a given key """
@abc.abstractmethod
def cleanup_service_tree(self):
""" clean all contents stored in the tree used for the tests """
@@ -374,7 +360,6 @@ class ConsulController(AbstractDcsController):
def __init__(self, context):
super(ConsulController, self).__init__(context)
os.environ['PATRONI_CONSUL_HOST'] = 'localhost:8500'
os.environ['PATRONI_CONSUL_REGISTER_SERVICE'] = 'on'
self._client = consul.Consul()
self._config_file = None
@@ -403,6 +388,9 @@ class ConsulController(AbstractDcsController):
_, value = self._client.kv.get(self.path(key, scope))
return value and value['Value'].decode('utf-8')
def set(self, key, value):
self._client.kv.put(self.path(key), value)
def cleanup_service_tree(self):
self._client.kv.delete(self.path(scope=''), recurse=True)
@@ -429,6 +417,9 @@ class EtcdController(AbstractDcsController):
except etcd.EtcdKeyNotFound:
return None
def set(self, key, value):
self._client.set(self.path(key), value)
def cleanup_service_tree(self):
try:
self._client.delete(self.path(scope=''), recursive=True)
@@ -445,76 +436,6 @@ class EtcdController(AbstractDcsController):
return False
class KubernetesController(AbstractDcsController):
def __init__(self, context):
super(KubernetesController, self).__init__(context)
self._namespace = 'default'
self._labels = {"application": "patroni"}
self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
os.environ['PATRONI_KUBERNETES_LABELS'] = json.dumps(self._labels)
os.environ['PATRONI_KUBERNETES_USE_ENDPOINTS'] = 'true'
from kubernetes import client as k8s_client, config as k8s_config
k8s_config.load_kube_config(context='local')
self._client = k8s_client
self._api = self._client.CoreV1Api()
def _start(self):
pass
def create_pod(self, name, scope):
labels = self._labels.copy()
labels['cluster-name'] = scope
metadata = self._client.V1ObjectMeta(namespace=self._namespace, name=name, labels=labels)
spec = self._client.V1PodSpec(containers=[self._client.V1Container(name=name, image='empty')])
body = self._client.V1Pod(metadata=metadata, spec=spec)
self._api.create_namespaced_pod(self._namespace, body)
def delete_pod(self, name):
try:
self._api.delete_namespaced_pod(name, self._namespace, self._client.V1DeleteOptions())
except:
pass
while True:
try:
self._api.read_namespaced_pod(name, self._namespace)
except:
break
def query(self, key, scope='batman'):
if key.startswith('members/'):
pod = self._api.read_namespaced_pod(key[8:], self._namespace)
return (pod.metadata.annotations or {}).get('status', '')
else:
try:
e = self._api.read_namespaced_endpoints(scope + ('' if key == 'leader' else '-' + key), self._namespace)
if key == 'leader':
return e.metadata.annotations[key]
else:
return json.dumps(e.metadata.annotations)
except:
return None
def cleanup_service_tree(self):
try:
self._api.delete_collection_namespaced_pod(self._namespace, label_selector=self._label_selector)
except:
pass
try:
self._api.delete_collection_namespaced_endpoints(self._namespace, label_selector=self._label_selector)
except:
pass
while True:
result = self._api.list_namespaced_pod(self._namespace, label_selector=self._label_selector)
if len(result.items) < 1:
break
def _is_running(self):
return True
class ZooKeeperController(AbstractDcsController):
""" handles all zookeeper related tasks, used for the tests setup and cleanup """
@@ -534,6 +455,9 @@ class ZooKeeperController(AbstractDcsController):
except kazoo.exceptions.NoNodeError:
return None
def set(self, key, value):
self._client.set(self.path(key), value.encode('utf-8'))
def cleanup_service_tree(self):
try:
self._client.delete(self.path(scope=''), recursive=True)
@@ -624,13 +548,6 @@ class PatroniPoolController(object):
'method': 'pg_basebackup',
'pg_basebackup': {
'command': self.BACKUP_SCRIPT + ' --walmethod=stream --dbname=' + f.backup_source
},
'dcs': {
'postgresql': {
'parameters': {
'max_connections': 101
}
}
}
},
'postgresql': {
@@ -796,7 +713,6 @@ class WatchdogMonitor(object):
# actions to execute on start/stop of the tests and before running invidual features
def before_all(context):
os.environ.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'})
context.ci = 'TRAVIS_BUILD_NUMBER' in os.environ or 'BUILD_NUMBER' in os.environ
context.timeout_multiplier = 2 if context.ci else 1
context.pctl = PatroniPoolController(context)
@@ -811,8 +727,8 @@ def before_all(context):
def after_all(context):
context.dcs_ctl.stop()
subprocess.call([sys.executable, '-m', 'coverage', 'combine'])
subprocess.call([sys.executable, '-m', 'coverage', 'report'])
subprocess.call(['coverage', 'combine'])
subprocess.call(['coverage', 'report'])
def before_feature(context, feature):
@@ -825,5 +741,3 @@ def after_feature(context, feature):
context.pctl.stop_all()
shutil.rmtree(os.path.join(context.pctl.patroni_path, 'data'))
context.dcs_ctl.cleanup_service_tree()
if feature.status == 'failed':
shutil.copytree(context.pctl.output_dir, context.pctl.output_dir + '_failed')
+17 -33
View File
@@ -13,17 +13,17 @@ Scenario: check API requests on a stand-alone server
When I run patronictl.py reinit batman postgres0 --force
Then I receive a response returncode 0
And I receive a response output "Failed: reinitialize for member postgres0, status code=503, (I am the leader, can not reinitialize)"
When I run patronictl.py switchover batman --master postgres0 --force
When I run patronictl.py failover batman --master postgres0 --force
Then I receive a response returncode 1
And I receive a response output "Error: No candidates found to switchover to"
When I issue a POST request to http://127.0.0.1:8008/switchover with {"leader": "postgres0"}
Then I receive a response code 412
And I receive a response text switchover is not possible: cluster does not have members except leader
And I receive a response output "Error: No candidates found to failover to"
When I issue a POST request to http://127.0.0.1:8008/failover with {"leader": "postgres0"}
Then I receive a response code 500
And I receive a response text failover is not possible: cluster does not have members except leader
When I issue an empty POST request to http://127.0.0.1:8008/failover
Then I receive a response code 400
When I issue a POST request to http://127.0.0.1:8008/failover with {"foo": "bar"}
Then I receive a response code 400
And I receive a response text "Failover could be performed only to a specific candidate"
And I receive a response text "No values given for required parameters leader and candidate"
Scenario: check local configuration reload
Given I issue an empty POST request to http://127.0.0.1:8008/reload
@@ -34,16 +34,16 @@ Scenario: check local configuration reload
Then I receive a response code 202
Scenario: check dynamic configuration change via DCS
Given I run patronictl.py edit-config -s 'ttl=10' -s 'loop_wait=2' -p 'max_connections=101' --force batman
Then I receive a response returncode 0
And I receive a response output "+loop_wait: 2"
Given I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 10, "loop_wait": 2, "postgresql": {"parameters": {"max_connections": 101}}}
Then I receive a response code 200
And I receive a response loop_wait 2
And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 11 seconds
When I issue a GET request to http://127.0.0.1:8008/config
Then I receive a response code 200
And I receive a response loop_wait 2
When I issue a GET request to http://127.0.0.1:8008/patroni
Then I receive a response code 200
And I receive a response tags {'new_tag': 'new_value'}
And I receive a response tags {'tag': 'new_value'}
Scenario: check API requests for the primary-replica pair in the pause mode
Given I run patronictl.py pause batman
@@ -64,42 +64,26 @@ Scenario: check API requests for the primary-replica pair in the pause mode
When I sleep for 10 seconds
Then postgres1 role is the secondary after 15 seconds
Scenario: check the switchover via the API in the pause mode
Given I issue a POST request to http://127.0.0.1:8008/switchover with {"leader": "postgres0", "candidate": "postgres1"}
Then I receive a response code 200
Scenario: check the failover via the API in the pause mode
Given I run patronictl.py failover batman --master postgres0 --candidate postgres1 --force
Then I receive a response returncode 0
And postgres1 is a leader after 5 seconds
And postgres1 role is the primary after 10 seconds
And postgres0 role is the secondary after 10 seconds
And replication works from postgres1 to postgres0 after 20 seconds
When I issue a GET request to http://127.0.0.1:8008/master
Then I receive a response code 503
When I issue a GET request to http://127.0.0.1:8008/replica
Then I receive a response code 200
When I issue a GET request to http://127.0.0.1:8009/master
Then I receive a response code 200
When I issue a GET request to http://127.0.0.1:8009/replica
Then I receive a response code 503
Scenario: check the scheduled switchover
Given I issue a scheduled switchover from postgres1 to postgres0 in 3 seconds
Scenario: check the scheduled failover
Given I issue a scheduled failover from postgres1 to postgres0 in 3 seconds
Then I receive a response returncode 1
And I receive a response output "Can't schedule switchover in the paused state"
And I receive a response output "Can't schedule failover in the paused state"
When I run patronictl.py resume batman
Then I receive a response returncode 0
Given I issue a scheduled switchover from postgres1 to postgres0 in 3 seconds
Given I issue a scheduled failover from postgres1 to postgres0 in 3 seconds
Then I receive a response returncode 0
And postgres0 is a leader after 20 seconds
And postgres0 role is the primary after 10 seconds
And postgres1 role is the secondary after 10 seconds
And replication works from postgres0 to postgres1 after 25 seconds
When I issue a GET request to http://127.0.0.1:8008/master
Then I receive a response code 200
When I issue a GET request to http://127.0.0.1:8008/replica
Then I receive a response code 503
When I issue a GET request to http://127.0.0.1:8009/master
Then I receive a response code 503
When I issue a GET request to http://127.0.0.1:8009/replica
Then I receive a response code 200
Scenario: check the scheduled restart
Given I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"superuser_reserved_connections": "6"}}}
-34
View File
@@ -1,34 +0,0 @@
Feature: standby cluster
Scenario: check permanent logical slots are preserved on failover/switchover
Given I start postgres1
Then postgres1 is a leader after 10 seconds
And I sleep for 2 seconds
When I issue a PATCH request to http://127.0.0.1:8009/config with {"loop_wait": 2, "slots": {"pm_1": {"type": "physical"}}, "postgresql": {"parameters": {"wal_level": "logical"}}}
Then I receive a response code 200
And Response on GET http://127.0.0.1:8009/config contains slots after 10 seconds
And I sleep for 2 seconds
When I issue a PATCH request to http://127.0.0.1:8009/config with {"slots": {"test_logical": {"type": "logical", "database": "postgres", "plugin": "test_decoding"}}}
Then I receive a response code 200
When I start postgres0 with callback configured
Then "members/postgres0" key in DCS has state=running after 10 seconds
And replication works from postgres1 to postgres0 after 15 seconds
When I shut down postgres1
Then postgres0 is a leader after 10 seconds
And I sleep for 2 seconds
When I issue a GET request to http://127.0.0.1:8008/
Then I receive a response code 200
And there is a label with "test_logical" in postgres0 data directory
Scenario: check replication of a single table in a standby cluster
Given I start postgres1 in a standby cluster batman1 as a clone of postgres0
Then postgres1 is a leader of batman1 after 10 seconds
When I add the table foo to postgres0
Then table foo is present on postgres1 after 20 seconds
When I start postgres2 in a cluster batman1
Then postgres2 role is the replica after 24 seconds
And table foo is present on postgres2 after 20 seconds
Scenario: check failover
When I kill postgres1
And I kill postmaster on postgres1
Then postgres2 is replicating from postgres0 after 20 seconds
+6 -10
View File
@@ -1,11 +1,9 @@
import base64
import json
import os
import parse
import requests
import shlex
import subprocess
import sys
import time
import yaml
@@ -80,13 +78,11 @@ def do_post_empty(context, url):
@step('I issue a {request_method:w} request to {url:url} with {data}')
def do_request(context, request_method, url, data):
data = data and json.loads(data) or {}
headers = {'Authorization': 'Basic ' + base64.b64encode('username:password'.encode('utf-8')).decode('utf-8'),
'Content-Type': 'application/json'}
try:
if request_method == 'PATCH':
r = requests.patch(url, headers=headers, json=data)
r = requests.patch(url, json=data)
else:
r = requests.post(url, headers=headers, json=data)
r = requests.post(url, json=data)
except requests.exceptions.RequestException:
context.status_code = None
context.response = None
@@ -96,7 +92,7 @@ def do_request(context, request_method, url, data):
@step('I run {cmd}')
def do_run(context, cmd):
cmd = [sys.executable, '-m', 'coverage', 'run', '--source=patroni', '-p'] + shlex.split(cmd)
cmd = ['coverage', 'run', '--source=patroni', '-p'] + shlex.split(cmd)
try:
# XXX: Dirty hack! We need to take name/passwd from the config!
env = os.environ.copy()
@@ -126,10 +122,10 @@ def check_response(context, component, data):
assert str(context.response[component]) == str(data), "{0} does not contain {1}".format(component, data)
@step('I issue a scheduled switchover from {from_host:w} to {to_host:w} in {in_seconds:d} seconds')
def scheduled_switchover(context, from_host, to_host, in_seconds):
@step('I issue a scheduled failover from {from_host:w} to {to_host:w} in {in_seconds:d} seconds')
def scheduled_failover(context, from_host, to_host, in_seconds):
context.execute_steps(u"""
Given I run patronictl.py switchover batman --master {0} --candidate {1} --scheduled "{2}" --force
Given I run patronictl.py failover batman --master {0} --candidate {1} --scheduled "{2}" --force
""".format(from_host, to_host, datetime.now(tzutc) + timedelta(seconds=int(in_seconds))))
-67
View File
@@ -1,67 +0,0 @@
import os
import time
from behave import step
select_replication_query = """
SELECT * FROM pg_catalog.pg_stat_replication
WHERE application_name = '{0}'
"""
@step('I start {name:w} with callback configured')
def start_patroni_with_callbacks(context, name):
return context.pctl.start(name, custom_config={
"postgresql": {
"callbacks": {
"on_role_change": "features/callback.sh"
}
}
})
@step('I start {name:w} in a cluster {cluster_name:w}')
def start_patroni(context, name, cluster_name):
return context.pctl.start(name, custom_config={
"scope": cluster_name
})
@step('I start {name:w} in a standby cluster {cluster_name:w} as a clone of {name2:w}')
def start_patroni_stanby_cluster(context, name, cluster_name, name2):
# we need to remove patroni.dynamic.json in order to "bootstrap" standby cluster with existing PGDATA
os.unlink(os.path.join(context.pctl._processes[name]._data_dir, 'patroni.dynamic.json'))
port = context.pctl._processes[name2]._connkwargs.get('port')
context.pctl._processes[name].update_config({
"scope": cluster_name,
"bootstrap": {
"dcs": {
"standby_cluster": {
"host": "localhost",
"port": port,
"primary_slot_name": "pm_1",
}
}
}
})
return context.pctl.start(name)
@step('{pg_name1:w} is replicating from {pg_name2:w} after {timeout:d} seconds')
def check_replication_status(context, pg_name1, pg_name2, timeout):
bound_time = time.time() + timeout
while time.time() < bound_time:
cur = context.pctl.query(
pg_name2,
select_replication_query.format(pg_name1),
fail_ok=True
)
if cur and len(cur.fetchall()) != 0:
return True
time.sleep(1)
return False
+1 -1
View File
@@ -31,7 +31,7 @@ def watchdog_was_closed(context, name):
assert context.pctl.get_watchdog(name).was_closed
@step('I reset {name:w} watchdog state')
@step('I wait for next {name:w} watchdog ping')
def watchdog_reset_pinged(context, name):
context.pctl.get_watchdog(name).reset()
+5 -17
View File
@@ -1,31 +1,19 @@
Feature: watchdog
Verify that watchdog gets pinged and triggered under appropriate circumstances.
Scenario: watchdog is opened and pinged
Scenario: watchdog is opened, pinged and closed
Given I start postgres0 with watchdog
Then postgres0 is a leader after 10 seconds
And postgres0 role is the primary after 10 seconds
And postgres0 watchdog has been pinged after 10 seconds
Scenario: watchdog is disabled during pause
Given I run patronictl.py pause batman
Then I receive a response returncode 0
When I sleep for 2 seconds
When I shut down postgres0
Then postgres0 watchdog has been closed
Scenario: watchdog is opened and pinged after resume
Given I reset postgres0 watchdog state
And I run patronictl.py resume batman
Then I receive a response returncode 0
And postgres0 watchdog has been pinged after 10 seconds
Scenario: watchdog is disabled when shutting down
Given I shut down postgres0
Then postgres0 watchdog has been closed
#TODO: test watchdog is disabled during pause
#TODO: test watchdog is disabled properly when shutting down
Scenario: watchdog is triggered if patroni stops responding
Given I reset postgres0 watchdog state
And I start postgres0 with watchdog
Given I start postgres0 with watchdog
Then postgres0 role is the primary after 10 seconds
When postgres0 hangs for 30 seconds
Then postgres0 watchdog is triggered after 30 seconds
-34
View File
@@ -1,34 +0,0 @@
FROM postgres:11
MAINTAINER Alexander Kukushkin <[email protected]>
RUN export DEBIAN_FRONTEND=noninteractive \
&& echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
&& apt-get update -y \
&& apt-get upgrade -y \
&& apt-cache depends patroni | sed -n -e 's/.* Depends: \(python3-.\+\)$/\1/p' \
| grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \
| xargs apt-get install -y vim-tiny curl jq locales git python3-pip python3-wheel \
## Make sure we have a en_US.UTF-8 locale available
&& localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
&& pip3 install setuptools \
&& pip3 install 'git+https://github.com/zalando/patroni.git#egg=patroni[kubernetes]' \
&& PGHOME=/home/postgres \
&& mkdir -p $PGHOME \
&& chown postgres $PGHOME \
&& sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \
# Set permissions for OpenShift
&& chmod 775 $PGHOME \
&& chmod 664 /etc/passwd \
# Clean up
&& apt-get remove -y git python3-pip python3-wheel \
&& apt-get autoremove -y \
&& apt-get clean -y \
&& rm -rf /var/lib/apt/lists/* /root/.cache
ADD entrypoint.sh /
EXPOSE 5432 8008
ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 EDITOR=/usr/bin/editor
USER postgres
WORKDIR /home/postgres
CMD ["/bin/bash", "/entrypoint.sh"]
-39
View File
@@ -1,39 +0,0 @@
#!/bin/bash
if [[ $UID -ge 10000 ]]; then
GID=$(id -g)
sed -e "s/^postgres:x:[^:]*:[^:]*:/postgres:x:$UID:$GID:/" /etc/passwd > /tmp/passwd
cat /tmp/passwd > /etc/passwd
rm /tmp/passwd
fi
cat > /home/postgres/patroni.yml <<__EOF__
bootstrap:
dcs:
postgresql:
use_pg_rewind: true
initdb:
- auth-host: md5
- auth-local: trust
- encoding: UTF8
- locale: en_US.UTF-8
- data-checksums
pg_hba:
- host all all 0.0.0.0/0 md5
- host replication ${PATRONI_REPLICATION_USERNAME} ${PATRONI_KUBERNETES_POD_IP}/16 md5
restapi:
connect_address: '${PATRONI_KUBERNETES_POD_IP}:8008'
postgresql:
connect_address: '${PATRONI_KUBERNETES_POD_IP}:5432'
authentication:
superuser:
password: '${PATRONI_SUPERUSER_PASSWORD}'
replication:
password: '${PATRONI_REPLICATION_PASSWORD}'
__EOF__
unset PATRONI_SUPERUSER_PASSWORD PATRONI_REPLICATION_PASSWORD
export KUBERNETES_NAMESPACE=$PATRONI_KUBERNETES_NAMESPACE
export POD_NAME=$PATRONI_NAME
exec /usr/bin/python3 /usr/local/bin/patroni /home/postgres/patroni.yml
-49
View File
@@ -1,49 +0,0 @@
# Patroni OpenShift Configuration
Patroni can be run in OpenShift. Based on the kubernetes configuration, the Dockerfile and Entrypoint has been modified to support the dynamic UID/GID configuration that is applied in OpenShift. This can be run under the standard `restricted` SCC.
# Examples
## Create test project
```
oc new-project patroni-test
```
## Build the image
Note: Update the references when merged upstream.
Note: If deploying as a template for multiple users, the following commands should be performed in a shared namespace like `openshift`.
```
oc import-image postgres:10 --confirm -n openshift
oc new-build https://github.com/zalando/patroni --context-dir=kubernetes -n openshift
```
## Deploy the Image
Two configuration templates exist in [templates](templates) directory:
- Patroni Ephemeral
- Patroni Persistent
The only difference is whether or not the statefulset requests persistent storage.
## Create the Template
Install the template into the `openshift` namespace if this should be shared across projects:
```
oc create -f templates/template_patroni_ephemeral.yml -n openshift
```
Then, from your own project:
```
oc new-app patroni-pgsql-ephemeral
```
Once the pods are running, two configmaps should be available:
```
$ oc get configmap
NAME DATA AGE
patroniocp-config 0 1m
patroniocp-leader 0 1m
```
@@ -1,287 +0,0 @@
apiVersion: v1
kind: Template
metadata:
name: patroni-pgsql-ephemeral
annotations:
description: |-
Patroni Postgresql database cluster, without persistent storage.
WARNING: Any data stored will be lost upon pod destruction. Only use this template for testing.
iconClass: icon-postgresql
openshift.io/display-name: Patroni Postgresql (Ephemeral)
openshift.io/long-description: This template deploys a a patroni postgresql HA cluster without persistent storage.
tags: postgresql
objects:
- apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
labels:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
name: ${PATRONI_CLUSTER_NAME}
spec:
ports:
- port: 5432
protocol: TCP
targetPort: 5432
sessionAffinity: None
type: ClusterIP
status:
loadBalancer: {}
- apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
labels:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
name: ${PATRONI_MASTER_SERVICE_NAME}
spec:
ports:
- port: 5432
protocol: TCP
targetPort: 5432
selector:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
role: master
sessionAffinity: None
type: ClusterIP
status:
loadBalancer: {}
- apiVersion: v1
kind: Secret
metadata:
name: ${PATRONI_CLUSTER_NAME}
labels:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
stringData:
superuser-password: ${PATRONI_SUPERUSER_PASSWORD}
replication-password: ${PATRONI_REPLICATION_PASSWORD}
- apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
labels:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
name: ${PATRONI_REPLICA_SERVICE_NAME}
spec:
ports:
- port: 5432
protocol: TCP
targetPort: 5432
selector:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
role: replica
sessionAffinity: None
type: ClusterIP
status:
loadBalancer: {}
- apiVersion: apps/v1
kind: StatefulSet
metadata:
creationTimestamp: null
generation: 3
labels:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
name: ${APPLICATION_NAME}
spec:
podManagementPolicy: OrderedReady
replicas: 3
revisionHistoryLimit: 10
selector:
matchLabels:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
serviceName: ${APPLICATION_NAME}
template:
metadata:
creationTimestamp: null
labels:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
spec:
containers:
- env:
- name: PATRONI_KUBERNETES_POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: PATRONI_KUBERNETES_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: PATRONI_KUBERNETES_LABELS
value: '{application: ${APPLICATION_NAME}, cluster-name: ${PATRONI_CLUSTER_NAME}}'
- name: PATRONI_SUPERUSER_USERNAME
value: ${PATRONI_SUPERUSER_USERNAME}
- name: PATRONI_SUPERUSER_PASSWORD
valueFrom:
secretKeyRef:
key: superuser-password
name: ${PATRONI_CLUSTER_NAME}
- name: PATRONI_REPLICATION_USERNAME
value: ${PATRONI_REPLICATION_USERNAME}
- name: PATRONI_REPLICATION_PASSWORD
valueFrom:
secretKeyRef:
key: replication-password
name: ${PATRONI_CLUSTER_NAME}
- name: PATRONI_SCOPE
value: ${PATRONI_CLUSTER_NAME}
- name: PATRONI_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: PATRONI_POSTGRESQL_DATA_DIR
value: /home/postgres/pgdata/pgroot/data
- name: PATRONI_POSTGRESQL_PGPASS
value: /tmp/pgpass
- name: PATRONI_POSTGRESQL_LISTEN
value: 0.0.0.0:5432
- name: PATRONI_RESTAPI_LISTEN
value: 0.0.0.0:8008
image: docker-registry.default.svc:5000/${NAMESPACE}/patroni:latest
imagePullPolicy: IfNotPresent
name: ${APPLICATION_NAME}
ports:
- containerPort: 8008
protocol: TCP
- containerPort: 5432
protocol: TCP
resources: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /home/postgres/pgdata
name: pgdata
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: ${SERVICE_ACCOUNT}
serviceAccountName: ${SERVICE_ACCOUNT}
terminationGracePeriodSeconds: 0
volumes:
- name: pgdata
emptyDir: {}
updateStrategy:
type: OnDelete
- apiVersion: v1
kind: Endpoints
metadata:
name: ${APPLICATION_NAME}
labels:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
subsets: []
- apiVersion: v1
kind: ServiceAccount
metadata:
name: ${SERVICE_ACCOUNT}
- apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: ${SERVICE_ACCOUNT}
rules:
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- get
- list
- patch
- update
- watch
# delete is required only for 'patronictl remove'
- delete
- apiGroups:
- ""
resources:
- endpoints
verbs:
- get
- patch
- update
# the following three privileges are necessary only when using endpoints
- create
- list
- watch
# delete is required only for for 'patronictl remove'
- delete
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- list
- patch
- update
- watch
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: ${SERVICE_ACCOUNT}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ${SERVICE_ACCOUNT}
subjects:
- kind: ServiceAccount
name: ${SERVICE_ACCOUNT}
parameters:
- description: The name of the application for labelling all artifacts.
displayName: Application Name
name: APPLICATION_NAME
value: patroni-ephemeral
- description: The name of the patroni-pgsql cluster.
displayName: Cluster Name
name: PATRONI_CLUSTER_NAME
value: patroni-ephemeral
- description: The name of the OpenShift Service exposed for the patroni-ephemeral-master container.
displayName: Master service name.
name: PATRONI_MASTER_SERVICE_NAME
value: patroni-ephemeral-master
- description: The name of the OpenShift Service exposed for the patroni-ephemeral-replica containers.
displayName: Replica service name.
name: PATRONI_REPLICA_SERVICE_NAME
value: patroni-ephemeral-replica
- description: Maximum amount of memory the container can use.
displayName: Memory Limit
name: MEMORY_LIMIT
value: 512Mi
- description: The OpenShift Namespace where the patroni and postgresql ImageStream resides.
displayName: ImageStream Namespace
name: NAMESPACE
value: openshift
- description: Username of the superuser account for initialization.
displayName: Superuser Username
name: PATRONI_SUPERUSER_USERNAME
value: postgres
- description: Password of the superuser account for initialization.
displayName: Superuser Passsword
name: PATRONI_SUPERUSER_PASSWORD
value: postgres
- description: Username of the replication account for initialization.
displayName: Replication Username
name: PATRONI_REPLICATION_USERNAME
value: postgres
- description: Password of the replication account for initialization.
displayName: Repication Passsword
name: PATRONI_REPLICATION_PASSWORD
value: postgres
- description: Service account name used for pods and rolebindings to form a cluster in the project.
displayName: Service Account
name: SERVICE_ACCOUNT
value: patroniocp
@@ -1,303 +0,0 @@
apiVersion: v1
kind: Template
metadata:
name: patroni-pgsql-persistent
annotations:
description: |-
Patroni Postgresql database cluster, with persistent storage.
WARNING: Any data stored will be lost upon pod destruction. Only use this template for testing.
iconClass: icon-postgresql
openshift.io/display-name: Patroni Postgresql (Persistent)
openshift.io/long-description: This template deploys a a patroni postgresql HA cluster without persistent storage.
tags: postgresql
objects:
- apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
labels:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
name: ${PATRONI_CLUSTER_NAME}
spec:
ports:
- port: 5432
protocol: TCP
targetPort: 5432
sessionAffinity: None
type: ClusterIP
status:
loadBalancer: {}
- apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
labels:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
name: ${PATRONI_MASTER_SERVICE_NAME}
spec:
ports:
- port: 5432
protocol: TCP
targetPort: 5432
selector:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
role: master
sessionAffinity: None
type: ClusterIP
status:
loadBalancer: {}
- apiVersion: v1
kind: Secret
metadata:
name: ${PATRONI_CLUSTER_NAME}
labels:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
stringData:
superuser-password: ${PATRONI_SUPERUSER_PASSWORD}
replication-password: ${PATRONI_REPLICATION_PASSWORD}
- apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
labels:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
name: ${PATRONI_REPLICA_SERVICE_NAME}
spec:
ports:
- port: 5432
protocol: TCP
targetPort: 5432
selector:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
role: replica
sessionAffinity: None
type: ClusterIP
status:
loadBalancer: {}
- apiVersion: apps/v1
kind: StatefulSet
metadata:
creationTimestamp: null
generation: 3
labels:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
name: ${APPLICATION_NAME}
spec:
podManagementPolicy: OrderedReady
replicas: 3
revisionHistoryLimit: 10
selector:
matchLabels:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
serviceName: ${APPLICATION_NAME}
template:
metadata:
creationTimestamp: null
labels:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
spec:
containers:
- env:
- name: PATRONI_KUBERNETES_POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: PATRONI_KUBERNETES_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: PATRONI_KUBERNETES_LABELS
value: '{application: ${APPLICATION_NAME}, cluster-name: ${PATRONI_CLUSTER_NAME}}'
- name: PATRONI_SUPERUSER_USERNAME
value: ${PATRONI_SUPERUSER_USERNAME}
- name: PATRONI_SUPERUSER_PASSWORD
valueFrom:
secretKeyRef:
key: superuser-password
name: ${PATRONI_CLUSTER_NAME}
- name: PATRONI_REPLICATION_USERNAME
value: ${PATRONI_REPLICATION_USERNAME}
- name: PATRONI_REPLICATION_PASSWORD
valueFrom:
secretKeyRef:
key: replication-password
name: ${PATRONI_CLUSTER_NAME}
- name: PATRONI_SCOPE
value: ${PATRONI_CLUSTER_NAME}
- name: PATRONI_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: PATRONI_POSTGRESQL_DATA_DIR
value: /home/postgres/pgdata/pgroot/data
- name: PATRONI_POSTGRESQL_PGPASS
value: /tmp/pgpass
- name: PATRONI_POSTGRESQL_LISTEN
value: 0.0.0.0:5432
- name: PATRONI_RESTAPI_LISTEN
value: 0.0.0.0:8008
image: docker-registry.default.svc:5000/${NAMESPACE}/patroni:latest
imagePullPolicy: IfNotPresent
name: ${APPLICATION_NAME}
ports:
- containerPort: 8008
protocol: TCP
- containerPort: 5432
protocol: TCP
resources: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /home/postgres/pgdata
name: ${APPLICATION_NAME}
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: ${SERVICE_ACCOUNT}
serviceAccountName: ${SERVICE_ACCOUNT}
terminationGracePeriodSeconds: 0
volumes:
- name: ${APPLICATION_NAME}
persistentVolumeClaim:
claimName: ${APPLICATION_NAME}
volumeClaimTemplates:
- metadata:
labels:
application: ${APPLICATION_NAME}
name: ${APPLICATION_NAME}
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: ${PVC_SIZE}
updateStrategy:
type: OnDelete
- apiVersion: v1
kind: Endpoints
metadata:
name: ${APPLICATION_NAME}
labels:
application: ${APPLICATION_NAME}
cluster-name: ${PATRONI_CLUSTER_NAME}
subsets: []
- apiVersion: v1
kind: ServiceAccount
metadata:
name: ${SERVICE_ACCOUNT}
- apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: ${SERVICE_ACCOUNT}
rules:
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- get
- list
- patch
- update
- watch
# delete is required only for 'patronictl remove'
- delete
- apiGroups:
- ""
resources:
- endpoints
verbs:
- get
- patch
- update
# the following three privileges are necessary only when using endpoints
- create
- list
- watch
# delete is required only for for 'patronictl remove'
- delete
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- list
- patch
- update
- watch
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: ${SERVICE_ACCOUNT}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ${SERVICE_ACCOUNT}
subjects:
- kind: ServiceAccount
name: ${SERVICE_ACCOUNT}
parameters:
- description: The name of the application for labelling all artifacts.
displayName: Application Name
name: APPLICATION_NAME
value: patroni-persistent
- description: The name of the patroni-pgsql cluster.
displayName: Cluster Name
name: PATRONI_CLUSTER_NAME
value: patroni-persistent
- description: The name of the OpenShift Service exposed for the patroni-persistent-master container.
displayName: Master service name.
name: PATRONI_MASTER_SERVICE_NAME
value: patroni-persistent-master
- description: The name of the OpenShift Service exposed for the patroni-persistent-replica containers.
displayName: Replica service name.
name: PATRONI_REPLICA_SERVICE_NAME
value: patroni-persistent-replica
- description: Maximum amount of memory the container can use.
displayName: Memory Limit
name: MEMORY_LIMIT
value: 512Mi
- description: The OpenShift Namespace where the patroni and postgresql ImageStream resides.
displayName: ImageStream Namespace
name: NAMESPACE
value: openshift
- description: Username of the superuser account for initialization.
displayName: Superuser Username
name: PATRONI_SUPERUSER_USERNAME
value: postgres
- description: Password of the superuser account for initialization.
displayName: Superuser Passsword
name: PATRONI_SUPERUSER_PASSWORD
value: postgres
- description: Username of the replication account for initialization.
displayName: Replication Username
name: PATRONI_REPLICATION_USERNAME
value: postgres
- description: Password of the replication account for initialization.
displayName: Repication Passsword
name: PATRONI_REPLICATION_PASSWORD
value: postgres
- description: Service account name used for pods and rolebindings to form a cluster in the project.
displayName: Service Account
name: SERVICE_ACCOUNT
value: patroni-persistent
- description: The size of the persistent volume to create.
displayName: Persistent Volume Size
name: PVC_SIZE
value: 5Gi
-43
View File
@@ -1,43 +0,0 @@
pipeline {
agent any
stages {
stage ('Deploy test pod'){
when {
expression {
openshift.withCluster() {
openshift.withProject() {
return !openshift.selector( "dc", "pgbench" ).exists()
}
}
}
}
steps {
script {
openshift.withCluster() {
openshift.withProject() {
def pgbench = openshift.newApp( "https://github.com/stewartshea/docker-pgbench/", "--name=pgbench", "-e PGPASSWORD=postgres", "-e PGUSER=postgres", "-e PGHOST=patroni-persistent-master", "-e PGDATABASE=postgres", "-e TEST_CLIENT_COUNT=20", "-e TEST_DURATION=120" )
def pgbenchdc = openshift.selector( "dc", "pgbench" )
timeout(5) {
pgbenchdc.rollout().status()
}
}
}
}
}
}
stage ('Run benchmark Test'){
steps {
sh '''
oc exec $(oc get pods -l app=pgbench | grep Running | awk '{print $1}') ./test.sh
'''
}
}
stage ('Clean up pgtest pod'){
steps {
sh '''
oc delete all -l app=pgbench
'''
}
}
}
}
@@ -1,2 +0,0 @@
# Jenkins Test
This pipeline test will create a separate deployment config for a pgbench pod and execute a test against the patroni cluster. This is a sample and should be customized.
-210
View File
@@ -1,210 +0,0 @@
# headless service to avoid deletion of patronidemo-config endpoint
apiVersion: v1
kind: Service
metadata:
name: patronidemo-config
labels:
application: patroni
cluster-name: patronidemo
spec:
clusterIP: None
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: &cluster_name patronidemo
labels:
application: patroni
cluster-name: *cluster_name
spec:
replicas: 3
serviceName: *cluster_name
template:
metadata:
labels:
application: patroni
cluster-name: *cluster_name
spec:
serviceAccountName: patronidemo
containers:
- name: *cluster_name
image: patroni # docker build -t patroni .
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8008
protocol: TCP
- containerPort: 5432
protocol: TCP
volumeMounts:
- mountPath: /home/postgres/pgdata
name: pgdata
env:
- name: PATRONI_KUBERNETES_POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: PATRONI_KUBERNETES_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: PATRONI_KUBERNETES_USE_ENDPOINTS
value: 'true'
- name: PATRONI_KUBERNETES_LABELS
value: '{application: patroni, cluster-name: patronidemo}'
- name: PATRONI_SUPERUSER_USERNAME
value: postgres
- name: PATRONI_SUPERUSER_PASSWORD
valueFrom:
secretKeyRef:
name: *cluster_name
key: superuser-password
- name: PATRONI_REPLICATION_USERNAME
value: standby
- name: PATRONI_REPLICATION_PASSWORD
valueFrom:
secretKeyRef:
name: *cluster_name
key: replication-password
- name: PATRONI_SCOPE
value: *cluster_name
- name: PATRONI_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: PATRONI_POSTGRESQL_DATA_DIR
value: /home/postgres/pgdata/pgroot/data
- name: PATRONI_POSTGRESQL_PGPASS
value: /tmp/pgpass
- name: PATRONI_POSTGRESQL_LISTEN
value: '0.0.0.0:5432'
- name: PATRONI_RESTAPI_LISTEN
value: '0.0.0.0:8008'
terminationGracePeriodSeconds: 0
volumes:
- name: pgdata
emptyDir: {}
# volumeClaimTemplates:
# - metadata:
# labels:
# application: spilo
# spilo-cluster: *cluster_name
# annotations:
# volume.alpha.kubernetes.io/storage-class: anything
# name: pgdata
# spec:
# accessModes:
# - ReadWriteOnce
# resources:
# requests:
# storage: 5Gi
---
apiVersion: v1
kind: Endpoints
metadata:
name: &cluster_name patronidemo
labels:
application: patroni
cluster-name: *cluster_name
subsets: []
---
apiVersion: v1
kind: Service
metadata:
name: &cluster_name patronidemo
labels:
application: patroni
cluster-name: *cluster_name
spec:
type: ClusterIP
ports:
- port: 5432
targetPort: 5432
---
apiVersion: v1
kind: Secret
metadata:
name: &cluster_name patronidemo
labels:
application: patroni
cluster-name: *cluster_name
type: Opaque
data:
superuser-password: emFsYW5kbw==
replication-password: cmVwLXBhc3M=
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: patronidemo
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: patronidemo
rules:
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- get
- list
- patch
- update
- watch
# delete is required only for 'patronictl remove'
- delete
- apiGroups:
- ""
resources:
- endpoints
verbs:
- get
- patch
- update
# the following three privileges are necessary only when using endpoints
- create
- list
- watch
# delete is required only for for 'patronictl remove'
- delete
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- list
- patch
- update
- watch
# The following privilege is only necessary for creation of headless service
# for patronidemo-config endpoint, in order to prevent cleaning it up by the
# k8s master. You can avoid giving this privilege by explicitly creating the
# service like it is done in this manifest (lines 2..10)
- apiGroups:
- ""
resources:
- services
verbs:
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: patronidemo
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: patronidemo
subjects:
- kind: ServiceAccount
name: patronidemo
+11 -22
View File
@@ -14,7 +14,6 @@ class Patroni(object):
from patroni.config import Config
from patroni.dcs import get_dcs
from patroni.ha import Ha
from patroni.log import PatroniLogger
from patroni.postgresql import Postgresql
from patroni.version import __version__
from patroni.watchdog import Watchdog
@@ -22,9 +21,7 @@ class Patroni(object):
self.setup_signal_handlers()
self.version = __version__
self.logger = PatroniLogger()
self.config = Config()
self.logger.reload_config(self.config.get('log', {}))
self.dcs = get_dcs(self.config)
self.watchdog = Watchdog(self.config)
self.load_dynamic_configuration()
@@ -42,7 +39,7 @@ class Patroni(object):
while True:
try:
cluster = self.dcs.get_cluster()
if cluster and cluster.config and cluster.config.data:
if cluster and cluster.config:
if self.config.set_dynamic_configuration(cluster.config):
self.dcs.reload_config(self.config)
self.watchdog.reload_config(self.config)
@@ -52,7 +49,6 @@ class Patroni(object):
break
except DCSError:
logger.warning('Can not get cluster from dcs')
time.sleep(5)
def get_tags(self):
return {tag: value for tag, value in self.config.get('tags', {}).items()
@@ -69,7 +65,6 @@ class Patroni(object):
def reload_config(self):
try:
self.tags = self.get_tags()
self.logger.reload_config(self.config.get('log', {}))
self.dcs.reload_config(self.config)
self.watchdog.reload_config(self.config)
self.api.reload_config(self.config['restapi'])
@@ -118,8 +113,8 @@ class Patroni(object):
logger.info(self.ha.run_cycle())
if self.dcs.cluster and self.dcs.cluster.config and self.dcs.cluster.config.data \
and self.config.set_dynamic_configuration(self.dcs.cluster.config):
cluster = self.dcs.cluster
if cluster and cluster.config and self.config.set_dynamic_configuration(cluster.config):
self.reload_config()
if self.postgresql.role != 'uninitialized':
@@ -130,19 +125,18 @@ class Patroni(object):
def setup_signal_handlers(self):
self._received_sighup = False
self._received_sigterm = False
if os.name != 'nt':
signal.signal(signal.SIGHUP, self.sighup_handler)
signal.signal(signal.SIGHUP, self.sighup_handler)
signal.signal(signal.SIGTERM, self.sigterm_handler)
def shutdown(self):
try:
self.api.shutdown()
except Exception:
logger.exception('Exception during RestApi.shutdown')
self.api.shutdown()
self.ha.shutdown()
def patroni_main():
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
logging.getLogger('requests').setLevel(logging.WARNING)
patroni = Patroni()
try:
patroni.run()
@@ -150,13 +144,10 @@ def patroni_main():
pass
finally:
patroni.shutdown()
logging.shutdown()
def pg_ctl_start(args):
import subprocess
if os.name != 'nt':
os.setsid()
postmaster = subprocess.Popen(args)
print(postmaster.pid)
@@ -191,7 +182,7 @@ def main():
if ret == (0, 0):
break
elif ret[0] != pid:
logger.info('Reaped pid=%s, exit status=%s', *ret)
logging.info('Reaped pid=%s, exit status=%s', *ret)
except OSError:
pass
@@ -200,13 +191,11 @@ def main():
os.kill(pid, signo)
signal.signal(signal.SIGCHLD, sigchld_handler)
if os.name != 'nt':
signal.signal(signal.SIGHUP, passtochild)
signal.signal(signal.SIGQUIT, passtochild)
signal.signal(signal.SIGHUP, passtochild)
signal.signal(signal.SIGINT, passtochild)
signal.signal(signal.SIGUSR1, passtochild)
signal.signal(signal.SIGUSR2, passtochild)
signal.signal(signal.SIGABRT, passtochild)
signal.signal(signal.SIGQUIT, passtochild)
signal.signal(signal.SIGTERM, passtochild)
patroni = call_self(sys.argv[1:])
+100 -160
View File
@@ -1,16 +1,14 @@
import base64
import fcntl
import json
import logging
import psycopg2
import time
import traceback
import dateutil.parser
import datetime
import os
from patroni.postgresql import PostgresConnectionException, PostgresException, Postgresql
from patroni.utils import deep_compare, parse_bool, patch_config, Retry, \
RetryFailedError, parse_int, split_host_port, tzutc
from patroni.exceptions import PostgresConnectionException
from patroni.utils import deep_compare, patch_config, Retry, RetryFailedError, is_valid_pg_version, parse_int, tzutc
from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
from six.moves.socketserver import ThreadingMixIn
from threading import Thread
@@ -26,9 +24,9 @@ def check_auth(func):
def do_PUT_foo():
pass
"""
def wrapper(handler, *args, **kwargs):
def wrapper(handler):
if handler.check_auth_header():
return func(handler, *args, **kwargs)
return func(handler)
return wrapper
@@ -72,8 +70,6 @@ class RestApiHandler(BaseHTTPRequestHandler):
response['scheduled_restart']['schedule'] = (response['scheduled_restart']['schedule']).isoformat()
if not patroni.ha.watchdog.is_healthy:
response['watchdog_failed'] = True
if patroni.ha.is_paused():
response['pause'] = True
self._write_json_response(status_code, response)
def do_GET(self, write_status_code_only=False):
@@ -84,27 +80,24 @@ class RestApiHandler(BaseHTTPRequestHandler):
patroni = self.server.patroni
cluster = patroni.dcs.cluster
replica_status_code = 200 if not patroni.noloadbalance and response.get('role') == 'replica' else 503
status_code = 503
if patroni.ha.is_standby_cluster() and ('standby_leader' in path or 'standby-leader' in path):
status_code = 200 if patroni.ha.is_leader() else 503
elif 'master' in path or 'leader' in path or 'primary' in path:
# Round-robing across all masters in pause mode if DCS is not accessible
if not cluster and patroni.ha.is_paused():
status_code = 200 if response['role'] == 'master' else 503
if cluster: # dcs available
if cluster.leader and cluster.leader.name == patroni.postgresql.name: # is_leader
status_code = 200 if 'master' in path else 503
elif 'role' not in response:
status_code = 503
elif response['role'] == 'master': # running as master but without leader lock!!!!
status_code = 503
elif response['role'] in path: # response['role'] != 'master'
status_code = 503 if patroni.noloadbalance else 200
else:
status_code = 200 if patroni.ha.is_leader() else 503
elif 'replica' in path:
status_code = replica_status_code
elif cluster: # dcs is available
is_synchronous = cluster.is_synchronous_mode() and cluster.sync \
and cluster.sync.sync_standby == patroni.postgresql.name
if path in ('/sync', '/synchronous') and is_synchronous:
status_code = replica_status_code
elif path in ('/async', '/asynchronous') and not is_synchronous:
status_code = replica_status_code
status_code = 503
elif 'role' in response and response['role'] in path:
status_code = 503 if response['role'] != 'master' and patroni.noloadbalance else 200
elif patroni.ha.restart_scheduled() and patroni.postgresql.role == 'master' and 'master' in path:
# exceptional case for master node when the postgres is being restarted via API
status_code = 200
else:
status_code = 503
if write_status_code_only: # when haproxy sends OPTIONS request it reads only status code and nothing more
message = self.responses[status_code][0]
@@ -228,11 +221,9 @@ class RestApiHandler(BaseHTTPRequestHandler):
data = "PostgreSQL role should be either master or replica"
break
elif k == 'postgres_version':
try:
Postgresql.postgres_version_to_int(request[k])
except PostgresException as e:
if not is_valid_pg_version(request[k]):
status_code = 400
data = e.value
data = "PostgreSQL version should be in the first.major.minor format"
break
elif k == 'timeout':
request[k] = parse_int(request[k], 's')
@@ -273,14 +264,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
@check_auth
def do_POST_reinitialize(self):
request = self._read_json_content(body_is_optional=True)
if request:
logger.debug('received reinitialize request: %s', request)
force = isinstance(request, dict) and parse_bool(request.get('force')) or False
data = self.server.patroni.ha.reinitialize(force)
data = self.server.patroni.ha.reinitialize()
if data is None:
status_code = 200
data = 'reinitialize started'
@@ -288,50 +272,43 @@ class RestApiHandler(BaseHTTPRequestHandler):
status_code = 503
self._write_response(status_code, data)
def poll_failover_result(self, leader, candidate, action):
def poll_failover_result(self, leader, candidate):
timeout = max(10, self.server.patroni.dcs.loop_wait)
for _ in range(0, timeout*2):
time.sleep(1)
try:
cluster = self.server.patroni.dcs.get_cluster()
if not cluster.is_unlocked() and cluster.leader.name != leader:
if cluster.leader and cluster.leader.name != leader:
if not candidate or candidate == cluster.leader.name:
return 200, 'Successfully {0}ed over to "{1}"'.format(action[:-4], cluster.leader.name)
return 200, 'Successfully failed over to "{0}"'.format(cluster.leader.name)
else:
return 200, '{0}ed over to "{1}" instead of "{2}"'.format(action[:-4].title(),
cluster.leader.name, candidate)
return 200, 'Failed over to "{0}" instead of "{1}"'.format(cluster.leader.name, candidate)
if not cluster.failover:
return 503, action.title() + ' failed'
return 503, 'Failover failed'
except Exception as e:
logger.debug('Exception occured during polling %s result: %s', action, e)
return 503, action.title() + ' status unknown'
logger.debug('Exception occured during polling failover result: %s', e)
return 503, 'Failover status unknown'
def is_failover_possible(self, cluster, leader, candidate, action):
def is_failover_possible(self, cluster, leader, candidate):
if leader and (not cluster.leader or cluster.leader.name != leader):
return 'leader name does not match'
if candidate:
if action == 'switchover' and cluster.is_synchronous_mode() and cluster.sync.sync_standby != candidate:
return 'candidate name does not match with sync_standby'
members = [m for m in cluster.members if m.name == candidate]
if not members:
return 'candidate does not exists'
elif cluster.is_synchronous_mode():
members = [m for m in cluster.members if m.name == cluster.sync.sync_standby]
if not members:
return action + ' is not possible: can not find sync_standby'
else:
members = [m for m in cluster.members if m.name != cluster.leader.name and m.api_url]
if not members:
return action + ' is not possible: cluster does not have members except leader'
for st in self.server.patroni.ha.fetch_nodes_statuses(members):
if st.failover_limitation() is None:
return 'failover is not possible: cluster does not have members except leader'
for _, reachable, _, _, tags in self.server.patroni.ha.fetch_nodes_statuses(members):
if reachable and not tags.get('nofailover', False):
return None
return action + ' is not possible: no good candidates have been found'
return 'failover is not possible: no good candidates have been found'
@check_auth
def do_POST_failover(self, action='failover'):
def do_POST_failover(self):
request = self._read_json_content()
(status_code, data) = (400, '')
status_code = 500
if not request:
return
@@ -340,47 +317,39 @@ class RestApiHandler(BaseHTTPRequestHandler):
scheduled_at = request.get('scheduled_at')
cluster = self.server.patroni.dcs.get_cluster()
logger.info("received %s request with leader=%s candidate=%s scheduled_at=%s",
action, leader, candidate, scheduled_at)
if scheduled_at and cluster.is_paused():
self._write_response(status_code, "Can't schedule failover in the paused state")
if action == 'failover' and not candidate:
data = 'Failover could be performed only to a specific candidate'
elif action == 'switchover' and not leader:
data = 'Switchover could be performed only from a specific leader'
logger.info("received failover request with leader=%s candidate=%s scheduled_at=%s",
leader, candidate, scheduled_at)
if not data and scheduled_at:
if not leader:
data = 'Scheduled {0} is possible only from a specific leader'.format(action)
if not data and cluster.is_paused():
data = "Can't schedule {0} in the paused state".format(action)
if not data:
(status_code, data, scheduled_at) = self.parse_schedule(scheduled_at, action)
if not data and cluster.is_paused() and not candidate:
data = action.title() + ' is possible only to a specific candidate in a paused state'
if not data and not scheduled_at:
data = self.is_failover_possible(cluster, leader, candidate, action)
if data:
status_code = 412
if not data:
if self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
self.server.patroni.ha.wakeup()
if scheduled_at:
data = action.title() + ' scheduled'
data = ''
if leader or candidate:
if scheduled_at:
(_, data, scheduled_at) = self.parse_schedule(scheduled_at, "failover")
if _:
status_code = _
elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
self.server.patroni.ha.wakeup()
data = 'Failover scheduled'
status_code = 202
else:
status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name,
candidate, action)
data = 'failed to write failover key into DCS'
status_code = 503
else:
data = 'failed to write {0} key into DCS'.format(action)
status_code = 503
data = self.is_failover_possible(cluster, leader, candidate)
if not data:
if self.server.patroni.dcs.manual_failover(leader, candidate):
self.server.patroni.ha.wakeup()
status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, candidate)
else:
data = 'failed to write failover key into DCS'
status_code = 503
else:
status_code = 400
data = 'No values given for required parameters leader and candidate'
self._write_response(status_code, data)
def do_POST_switchover(self):
self.do_POST_failover(action='switchover')
def parse_request(self):
"""Override parse_request method to enrich basic functionality of `BaseHTTPRequestHandler` class
@@ -407,51 +376,40 @@ class RestApiHandler(BaseHTTPRequestHandler):
def get_postgresql_status(self, retry=False):
try:
cluster = self.server.patroni.dcs.cluster
if self.server.patroni.postgresql.state not in ('running', 'restarting', 'starting'):
raise RetryFailedError('')
stmt = ("WITH replication_info AS ("
"SELECT usename, application_name, client_addr, state, sync_state, sync_priority"
" FROM pg_catalog.pg_stat_replication) SELECT"
" pg_catalog.to_char(pg_catalog.pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),"
" CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0"
" ELSE ('x' || pg_catalog.substr(pg_catalog.pg_{0}file_name("
"pg_catalog.pg_current_{0}_{1}()), 1, 8))::bit(32)::int END,"
" CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0"
" ELSE pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}_{1}(), '0/0')::bigint END,"
" pg_catalog.pg_{0}_{1}_diff(COALESCE(pg_catalog.pg_last_{0}_receive_{1}(),"
" pg_catalog.pg_last_{0}_replay_{1}()), '0/0')::bigint,"
" pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), '0/0')::bigint,"
" pg_catalog.to_char(pg_catalog.pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),"
" pg_catalog.pg_is_in_recovery() AND pg_catalog.pg_is_{0}_replay_paused(), "
"(SELECT pg_catalog.array_to_json(pg_catalog.array_agg("
"pg_catalog.row_to_json(ri))) FROM replication_info ri)")
row = self.query(stmt.format(self.server.patroni.postgresql.wal_name,
self.server.patroni.postgresql.lsn_name), retry=retry)[0]
row = self.query("""WITH replication_info AS (
SELECT usename, application_name, client_addr, state, sync_state, sync_priority
FROM pg_stat_replication
)
SELECT to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
pg_is_in_recovery(),
CASE WHEN pg_is_in_recovery()
THEN 0
ELSE pg_{0}_{1}_diff(pg_current_{0}_{1}(), '0/0')::bigint
END,
pg_{0}_{1}_diff(COALESCE(pg_last_{0}_receive_{1}(),
pg_last_{0}_replay_{1}()), '0/0')::bigint,
pg_{0}_{1}_diff(pg_last_{0}_replay_{1}(), '0/0')::bigint,
to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
pg_is_in_recovery() AND pg_is_{0}_replay_paused(),
(SELECT array_to_json(array_agg(row_to_json(ri)))
FROM replication_info ri)""".format(self.server.patroni.postgresql.wal_name,
self.server.patroni.postgresql.lsn_name),
retry=retry)[0]
result = {
'state': self.server.patroni.postgresql.state,
'postmaster_start_time': row[0],
'role': 'replica' if row[1] == 0 else 'master',
'role': 'replica' if row[1] else 'master',
'server_version': self.server.patroni.postgresql.server_version,
'cluster_unlocked': bool(not cluster or cluster.is_unlocked()),
'xlog': ({
'received_location': row[3],
'replayed_location': row[4],
'replayed_timestamp': row[5],
'paused': row[6]} if row[1] == 0 else {
'paused': row[6]} if row[1] else {
'location': row[2]
})
}
if row[1] > 0:
result['timeline'] = row[1]
else:
leader_timeline = None if not cluster or cluster.is_unlocked() else cluster.leader.timeline
result['timeline'] = self.server.patroni.postgresql.replica_cached_timeline(leader_timeline)
if row[7]:
result['replication'] = row[7]
@@ -471,7 +429,6 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
def __init__(self, patroni, config):
self.patroni = patroni
self.__listen = None
self.__initialize(config)
self.__set_config_parameters(config)
self.daemon = True
@@ -489,10 +446,8 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
@staticmethod
def _set_fd_cloexec(fd):
if os.name != 'nt':
import fcntl
flags = fcntl.fcntl(fd, fcntl.F_GETFD)
fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC)
flags = fcntl.fcntl(fd, fcntl.F_GETFD)
fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC)
def check_basic_auth_key(self, key):
return self.__auth_key == key
@@ -508,25 +463,18 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
def __get_ssl_options(config):
return {option: config[option] for option in ['certfile', 'keyfile'] if option in config}
def __set_connection_string(self, connect_address):
self.connection_string = '{0}://{1}/patroni'.format(self.__protocol, connect_address or self.__listen)
def __set_config_parameters(self, config):
self.__auth_key = base64.b64encode(config['auth'].encode('utf-8')).decode('utf-8') if 'auth' in config else None
self.connection_string = '{0}://{1}/patroni'.format(self.__protocol,
config.get('connect_address') or self.__listen)
self.__set_connection_string(config.get('connect_address'))
def __initialize(self, config):
try:
host, port = split_host_port(config['listen'], None)
except Exception:
raise ValueError('Invalid "restapi" config: expected <HOST>:<PORT> for "listen", but got "{0}"'
.format(config['listen']))
if self.__listen is not None: # changing config in runtime
self.shutdown()
self.__listen = config['listen']
self.__ssl_options = self.__get_ssl_options(config)
HTTPServer.__init__(self, (host, port), RestApiHandler)
self.__listen = config['listen']
host, port = config['listen'].split(':')
HTTPServer.__init__(self, (host, int(port)), RestApiHandler)
Thread.__init__(self, target=self.serve_forever)
self._set_fd_cloexec(self.socket)
@@ -538,19 +486,11 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
import ssl
self.socket = ssl.wrap_socket(self.socket, server_side=True, **self.__ssl_options)
self.__protocol = 'https'
return True
self.__set_connection_string(config.get('connect_address'))
def reload_config(self, config):
if 'listen' not in config: # changing config in runtime
raise ValueError('Can not find "restapi.listen" config')
elif (self.__listen != config['listen'] or self.__ssl_options != self.__get_ssl_options(config)) \
and self.__initialize(config):
self.start()
self.__set_config_parameters(config)
@staticmethod
def handle_error(request, client_address):
address, port = client_address
logger.warning('Exception happened during processing of request from {}:{}'.format(address, port))
logger.warning(traceback.format_exc())
if self.__listen != config['listen'] or self.__ssl_options != self.__get_ssl_options(config):
self.shutdown()
self.__initialize(config)
self.start()
+4 -30
View File
@@ -1,5 +1,5 @@
import logging
from threading import Event, Lock, RLock, Thread
from threading import Lock, RLock, Thread
logger = logging.getLogger(__name__)
@@ -52,27 +52,22 @@ class CriticalTask(object):
class AsyncExecutor(object):
def __init__(self, state_handler, ha_wakeup):
self.state_handler = state_handler
def __init__(self, ha_wakeup):
self._ha_wakeup = ha_wakeup
self._thread_lock = RLock()
self._scheduled_action = None
self._scheduled_action_lock = RLock()
self._is_cancelled = False
self._finish_event = Event()
self.critical_task = CriticalTask()
@property
def busy(self):
return self.scheduled_action is not None
def schedule(self, action):
def schedule(self, action, immediately=False):
with self._scheduled_action_lock:
if self._scheduled_action is not None:
return self._scheduled_action
self._scheduled_action = action
self._is_cancelled = False
self._finish_event.set()
return None
@property
@@ -87,21 +82,14 @@ class AsyncExecutor(object):
def run(self, func, args=()):
wakeup = False
try:
with self:
if self._is_cancelled:
return
self._finish_event.clear()
self.state_handler.reset_is_cancelled()
# if the func returned something (not None) - wake up main HA loop
wakeup = func(*args) if args else func()
return wakeup
except Exception:
except:
logger.exception('Exception during execution of long running task %s', self.scheduled_action)
finally:
with self:
self.reset_scheduled_action()
self._finish_event.set()
with self.critical_task:
self.critical_task.reset()
if wakeup is not None:
@@ -110,20 +98,6 @@ class AsyncExecutor(object):
def run_async(self, func, args=()):
Thread(target=self.run, args=(func, args)).start()
def cancel(self):
with self:
with self._scheduled_action_lock:
if self._scheduled_action is None:
return
logger.warning('Cancelling long running task %s', self._scheduled_action)
self._is_cancelled = True
self.state_handler.cancel()
self._finish_event.wait()
with self:
self.reset_scheduled_action()
def __enter__(self):
self._thread_lock.acquire()
+28 -91
View File
@@ -1,7 +1,6 @@
import json
import logging
import os
import shutil
import sys
import tempfile
import yaml
@@ -10,8 +9,7 @@ from collections import defaultdict
from copy import deepcopy
from patroni.dcs import ClusterConfig
from patroni.postgresql import Postgresql
from patroni.utils import deep_compare, parse_bool, parse_int, patch_config
from requests.structures import CaseInsensitiveDict
from patroni.utils import deep_compare, parse_int, patch_config
logger = logging.getLogger(__name__)
@@ -43,23 +41,13 @@ class Config(object):
__DEFAULT_CONFIG = {
'ttl': 30, 'loop_wait': 10, 'retry_timeout': 10,
'maximum_lag_on_failover': 1048576,
'check_timeline': False,
'master_start_timeout': 300,
'synchronous_mode': False,
'synchronous_mode_strict': False,
'standby_cluster': {
'create_replica_methods': '',
'host': '',
'port': '',
'primary_slot_name': '',
'restore_command': '',
'archive_cleanup_command': '',
'recovery_min_apply_delay': ''
},
'postgresql': {
'bin_dir': '',
'use_slots': True,
'parameters': CaseInsensitiveDict({p: v[0] for p, v in Postgresql.CMDLINE_OPTIONS.items()})
'parameters': {p: v[0] for p, v in Postgresql.CMDLINE_OPTIONS.items()}
},
'watchdog': {
'mode': 'automatic',
@@ -99,9 +87,6 @@ class Config(object):
def dynamic_configuration(self):
return deepcopy(self._dynamic_configuration)
def check_mode(self, mode):
return bool(parse_bool(self._dynamic_configuration.get(mode)))
def _load_config_file(self):
"""Loads config.yaml from filesystem and applies some values which were set via ENV"""
with open(self._config_file) as f:
@@ -125,7 +110,7 @@ class Config(object):
with os.fdopen(fd, 'w') as f:
fd = None
json.dump(self.dynamic_configuration, f)
tmpfile = shutil.move(tmpfile, self._cache_file)
tmpfile = os.rename(tmpfile, self._cache_file)
self._cache_needs_saving = False
except Exception:
logger.exception('Exception when saving file: %s', self._cache_file)
@@ -169,8 +154,6 @@ class Config(object):
self._local_configuration = configuration
self.__effective_configuration = new_configuration
return True
else:
logger.info('No configuration items changed, nothing to reload.')
except Exception:
logger.exception('Exception when reloading local configuration from %s', self.config_file)
if dry_run:
@@ -194,10 +177,6 @@ class Config(object):
config['postgresql'][name].update(self._process_postgresql_parameters(value))
elif name not in ('connect_address', 'listen', 'data_dir', 'pgpass', 'authentication'):
config['postgresql'][name] = deepcopy(value)
elif name == 'standby_cluster':
for name, value in (value or {}).items():
if name in self.__DEFAULT_CONFIG['standby_cluster']:
config['standby_cluster'][name] = deepcopy(value)
elif name in config: # only variables present in __DEFAULT_CONFIG allowed to be overriden from DCS
if name in ('synchronous_mode', 'synchronous_mode_strict'):
config[name] = value
@@ -217,15 +196,6 @@ class Config(object):
if value:
ret[param] = value
def _fix_log_env(name, oldname):
value = _popenv(oldname)
name = Config.PATRONI_ENV_PREFIX + 'LOG_' + name.upper()
if value and name not in os.environ:
os.environ[name] = value
for name, oldname in (('level', 'loglevel'), ('format', 'logformat'), ('dateformat', 'log_datefmt')):
_fix_log_env(name, oldname)
def _set_section_values(section, params):
for param in params:
value = _popenv(section + '_' + param)
@@ -234,22 +204,6 @@ class Config(object):
_set_section_values('restapi', ['listen', 'connect_address', 'certfile', 'keyfile'])
_set_section_values('postgresql', ['listen', 'connect_address', 'data_dir', 'pgpass', 'bin_dir'])
_set_section_values('log', ['level', 'format', 'dateformat', 'dir', 'file_size', 'file_num', 'loggers'])
def _parse_dict(value):
if not value.strip().startswith('{'):
value = '{{{0}}}'.format(value)
try:
return yaml.safe_load(value)
except Exception:
logger.exception('Exception when parsing dict %s', value)
return None
value = ret.get('log', {}).pop('loggers', None)
if value:
value = _parse_dict(value)
if value:
ret['log']['loggers'] = value
def _get_auth(name):
ret = {}
@@ -272,6 +226,8 @@ class Config(object):
if authentication:
ret['postgresql']['authentication'] = authentication
users = {}
def _parse_list(value):
if not (value.strip().startswith('-') or '[' in value):
value = '[{0}]'.format(value)
@@ -281,42 +237,30 @@ class Config(object):
logger.exception('Exception when parsing list %s', value)
return None
for param in list(os.environ.keys()):
if param.startswith(Config.PATRONI_ENV_PREFIX):
# PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
name, suffix = (param[8:].split('_', 1) + [''])[:2]
if suffix in ('HOST', 'HOSTS', 'PORT', 'PROTOCOL', 'SRV', 'URL', 'PROXY', 'CACERT', 'CERT', 'KEY',
'VERIFY', 'TOKEN', 'CHECKS', 'DC', 'REGISTER_SERVICE', 'SERVICE_CHECK_INTERVAL',
'NAMESPACE', 'CONTEXT', 'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL', 'POD_IP',
'PORTS', 'LABELS') and name:
value = os.environ.pop(param)
if suffix == 'PORT':
value = value and parse_int(value)
elif suffix in ('HOSTS', 'PORTS', 'CHECKS'):
value = value and _parse_list(value)
elif suffix == 'LABELS':
value = _parse_dict(value)
elif suffix == 'REGISTER_SERVICE':
value = parse_bool(value)
if value:
ret[name.lower()][suffix.lower()] = value
if 'etcd' in ret:
ret['etcd'].update(_get_auth('etcd'))
users = {}
for param in list(os.environ.keys()):
if param.startswith(Config.PATRONI_ENV_PREFIX):
name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
# PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>
# CREATE USER "<username>" WITH <OPTIONS> PASSWORD '<password>'
if name and suffix == 'PASSWORD':
password = os.environ.pop(param)
if password:
users[name] = {'password': password}
options = os.environ.pop(param[:-9] + '_OPTIONS', None)
options = options and _parse_list(options)
if options:
users[name]['options'] = options
if name and suffix:
# PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
if suffix in ('HOST', 'HOSTS', 'PORT', 'SRV', 'URL', 'PROXY', 'CACERT', 'CERT', 'KEY') \
and '_' not in name:
value = os.environ.pop(param)
if suffix == 'PORT':
value = value and parse_int(value)
elif suffix == 'HOSTS':
value = value and _parse_list(value)
if value:
ret[name.lower()][suffix.lower()] = value
# PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>
# CREATE USER "<username>" WITH <OPTIONS> PASSWORD '<password>'
elif suffix == 'PASSWORD':
password = os.environ.pop(param)
if password:
users[name] = {'password': password}
options = os.environ.pop(param[:-9] + '_OPTIONS', None)
options = options and _parse_list(options)
if options:
users[name]['options'] = options
if users:
ret['bootstrap']['users'] = users
@@ -358,15 +302,8 @@ class Config(object):
if 'name' not in config and 'name' in pg_config:
config['name'] = pg_config['name']
updated_fields = (
'name',
'scope',
'retry_timeout',
'synchronous_mode',
'synchronous_mode_strict',
)
pg_config.update({p: config[p] for p in updated_fields if p in config})
pg_config.update({p: config[p] for p in ('name', 'scope', 'retry_timeout',
'synchronous_mode', 'maximum_lag_on_failover') if p in config})
return config
+106 -273
View File
@@ -30,8 +30,7 @@ from patroni.config import Config
from patroni.dcs import get_dcs as _get_dcs
from patroni.exceptions import PatroniException
from patroni.postgresql import Postgresql
from patroni.utils import patch_config, polling_loop
from patroni.version import __version__
from patroni.utils import is_valid_pg_version, patch_config
from prettytable import PrettyTable
from six.moves.urllib_parse import urlparse
from six import text_type
@@ -97,26 +96,19 @@ def store_config(config, path):
yaml.dump(config, fd)
option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, json, yaml)', default='pretty')
option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, json)', default='pretty')
option_watchrefresh = click.option('-w', '--watch', type=float, help='Auto update the screen every X seconds')
option_watch = click.option('-W', is_flag=True, help='Auto update the screen every 2 seconds')
option_force = click.option('--force', is_flag=True, help='Do not ask for confirmation at any point')
arg_cluster_name = click.argument('cluster_name', required=False,
default=lambda: click.get_current_context().obj.get('scope'))
option_insecure = click.option('-k', '--insecure', is_flag=True, help='Allow connections to SSL sites without certs')
@click.group()
@click.option('--config-file', '-c', help='Configuration file', default=CONFIG_FILE_PATH)
@click.option('--dcs', '-d', help='Use this DCS', envvar='DCS')
@option_insecure
@click.pass_context
def ctl(ctx, config_file, dcs, insecure):
def ctl(ctx, config_file, dcs):
logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=os.environ.get('LOGLEVEL', 'WARNING'))
logging.captureWarnings(True) # Capture eventual SSL warning
ctx.obj = load_config(config_file, dcs)
# backward compatibility for configuration file where ctl section is not define
ctx.obj.setdefault('ctl', {})['insecure'] = ctx.obj.get('ctl', {}).get('insecure') or insecure
def get_dcs(config, scope):
@@ -134,7 +126,6 @@ def auth_header(config):
def request_patroni(member, request_type, endpoint, content=None, headers=None):
ctx = click.get_current_context() # the current click context
headers = headers or {}
url_parts = urlparse(member.api_url)
logging.debug(url_parts)
@@ -143,21 +134,8 @@ def request_patroni(member, request_type, endpoint, content=None, headers=None):
url = '{0}://{1}/{2}'.format(url_parts.scheme, url_parts.netloc, endpoint)
insecure = ctx.obj.get('ctl', {}).get('insecure', False)
# Get certfile if any from several configuration namespace
cert = ctx.obj.get('ctl', {}).get('cacert') or \
ctx.obj.get('restapi', {}).get('cacert') or \
ctx.obj.get('restapi', {}).get('certfile')
# In the case we specificaly disable SSL cert verification we don't want to have the warning
if insecure:
verify = False
elif cert:
verify = cert
else:
verify = True
return getattr(requests, request_type)(url, headers=headers,
data=json.dumps(content) if content else None, timeout=60,
verify=verify)
data=json.dumps(content) if content else None, timeout=60)
def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True, delimiter='\t'):
@@ -171,16 +149,13 @@ def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True,
click.echo(t)
return
if fmt in ['json', 'yaml', 'yml']:
if fmt == 'json':
elements = [dict(zip(columns, r)) for r in rows]
if fmt == 'json':
click.echo(json.dumps(elements))
elif fmt in ('yaml', 'yml'):
click.echo(yaml.safe_dump(elements, encoding=None, default_flow_style=False, allow_unicode=True, width=200))
click.echo(json.dumps(elements))
if fmt == 'tsv':
if columns is not None and header:
click.echo(delimiter.join(columns))
click.echo(delimiter.join(columns) + '\n')
for r in rows:
c = [str(c) for c in r]
@@ -252,7 +227,7 @@ def get_cursor(cluster, connect_parameters, role='master', member=None):
if role == 'any':
return cursor
cursor.execute('SELECT pg_catalog.pg_is_in_recovery()')
cursor.execute('SELECT pg_is_in_recovery()')
in_recovery = cursor.fetchone()[0]
if in_recovery and role == 'replica' or not in_recovery and role == 'master':
@@ -282,9 +257,9 @@ def get_members(cluster, cluster_name, member_names, role, force, action):
member_names = [click.prompt('Which member do you want to {0} [{1}]?'.format(action,
', '.join(candidates.keys())), type=str, default='')]
for member_name in member_names:
if member_name not in candidates:
raise PatroniCtlException('{0} is not a member of cluster'.format(member_name))
for mn in member_names:
if mn not in candidates:
raise PatroniCtlException('{0} is not a member of cluster'.format(mn))
if not force:
confirm = click.confirm('Are you sure you want to {0} members {1}?'.format(action, ', '.join(member_names)))
@@ -298,7 +273,7 @@ def get_members(cluster, cluster_name, member_names, role, force, action):
@click.option('--role', '-r', help='Give a dsn of any member with this role', type=click.Choice(['master', 'replica',
'any']), default=None)
@click.option('--member', '-m', help='Generate a dsn for this member', type=str)
@arg_cluster_name
@click.argument('cluster_name')
@click.pass_obj
def dsn(obj, cluster_name, role, member):
if role is not None and member is not None:
@@ -316,7 +291,7 @@ def dsn(obj, cluster_name, role, member):
@ctl.command('query', help='Query a Patroni PostgreSQL member')
@arg_cluster_name
@click.argument('cluster_name')
@option_format
@click.option('--format', 'fmt', help='Output format (pretty, json)', default='tsv')
@click.option('--file', '-f', 'p_file', help='Execute the SQL commands from this file', type=click.File('rb'))
@@ -392,7 +367,7 @@ def query_member(cluster, cursor, member, role, command, connect_parameters):
logging.debug(message)
return [[timestamp(0), message]], None
cursor.execute('SELECT pg_catalog.pg_is_in_recovery()')
cursor.execute('SELECT pg_is_in_recovery()')
in_recovery = cursor.fetchone()[0]
if in_recovery and role == 'master' or not in_recovery and role == 'replica':
@@ -431,7 +406,7 @@ def remove(obj, cluster_name, fmt):
if message != confirm:
raise PatroniCtlException('You did not exactly type "{0}"'.format(message))
if cluster.leader and cluster.leader.name:
if cluster.leader:
confirm = click.prompt('This cluster currently is healthy. Please specify the master name to continue')
if confirm != cluster.leader.name:
raise PatroniCtlException('You did not specify the current master of the cluster')
@@ -444,10 +419,8 @@ def check_response(response, member_name, action_name, silent_success=False):
click.echo('Failed: {0} for member {1}, status code={2}, ({3})'.format(
action_name, member_name, response.status_code, response.text
))
return False
elif not silent_success:
click.echo('Success: {0} for member {1}'.format(action_name, member_name))
return True
def parse_scheduled(scheduled):
@@ -464,33 +437,6 @@ def parse_scheduled(scheduled):
return None
@ctl.command('reload', help='Reload cluster member configuration')
@click.argument('cluster_name')
@click.argument('member_names', nargs=-1)
@click.option('--role', '-r', help='Reload only members with this role', default='any',
type=click.Choice(['master', 'replica', 'any']))
@option_force
@click.pass_obj
def reload(obj, cluster_name, member_names, force, role):
cluster = get_dcs(obj, cluster_name).get_cluster()
members = get_members(cluster, cluster_name, member_names, role, force, 'reload')
content = {}
for member in members:
r = request_patroni(member, 'post', 'reload', content, auth_header(obj))
if r.status_code == 200:
click.echo('No changes to apply on member {0}'.format(member.name))
elif r.status_code == 202:
click.echo('Reload request received for member {0} and will be processed within {1} seconds'.format(
member.name, cluster.config.data.get('loop_wait'))
)
else:
click.echo('Failed: reload for member {0}, status code={1}, ({2})'.format(
member.name, r.status_code, r.text)
)
@ctl.command('restart', help='Restart cluster member')
@click.argument('cluster_name')
@click.argument('member_names', nargs=-1)
@@ -523,12 +469,11 @@ def restart(obj, cluster_name, member_names, force, role, p_any, scheduled, vers
content['restart_pending'] = True
if version:
try:
Postgresql.postgres_version_to_int(version)
except PatroniException as e:
raise PatroniCtlException(e.value)
content['postgres_version'] = version
if not is_valid_pg_version(version):
message = 'PostgreSQL version should be in the first.major.minor format'
raise PatroniCtlException(message)
else:
content['postgres_version'] = version
if scheduled is None and not force:
scheduled = click.prompt('When should the restart take place (e.g. 2015-10-01T14:30) ', type=str, default='now')
@@ -571,71 +516,68 @@ def reinit(obj, cluster_name, member_names, force):
members = get_members(cluster, cluster_name, member_names, None, force, 'reinitialize')
for member in members:
body = {'force': force}
while True:
r = request_patroni(member, 'post', 'reinitialize', body, auth_header(obj))
if not check_response(r, member.name, 'reinitialize') and r.text.endswith(' already in progress') \
and not force and click.confirm('Do you want to cancel it and reinitialize anyway?'):
body['force'] = True
continue
break
r = request_patroni(member, 'post', 'reinitialize', headers=auth_header(obj))
check_response(r, member.name, 'reinitialize')
def _do_failover_or_switchover(obj, action, cluster_name, master, candidate, force, scheduled=None):
@ctl.command('failover', help='Failover to a replica')
@click.argument('cluster_name')
@click.option('--master', help='The name of the current master', default=None)
@click.option('--candidate', help='The name of the candidate', default=None)
@click.option('--scheduled', help='Timestamp of a scheduled failover in unambiguous format (e.g. ISO 8601)',
default=None)
@option_force
@click.pass_obj
def failover(obj, cluster_name, master, candidate, force, scheduled):
"""
We want to trigger a failover or switchover for the specified cluster name.
We want to trigger a failover for the specified cluster name.
We verify that the cluster name, master name and candidate name are correct.
If so, we trigger an action and keep the client up to date.
If so, we trigger a failover and keep the client up to date.
"""
dcs = get_dcs(obj, cluster_name)
cluster = dcs.get_cluster()
if action == 'switchover' and cluster.leader is None:
if cluster.leader is None and not cluster.is_paused():
raise PatroniCtlException('This cluster has no master')
if master is None:
if force or action == 'failover':
master = cluster.leader and cluster.leader.name
if master is None and (not cluster.is_paused() or cluster.leader):
if force:
master = cluster.leader.member.name
else:
master = click.prompt('Master', type=str, default=cluster.leader.member.name)
if master is not None and cluster.leader and cluster.leader.member.name != master:
raise PatroniCtlException('Member {0} is not the leader of cluster {1}'.format(master, cluster_name))
# excluding members with nofailover tag
candidate_names = [str(m.name) for m in cluster.members if m.name != master and not m.nofailover]
candidate_names = [str(m.name) for m in cluster.members if m.name != master]
# We sort the names for consistent output to the client
candidate_names.sort()
if not candidate_names:
raise PatroniCtlException('No candidates found to {0} to'.format(action))
raise PatroniCtlException('No candidates found to failover to')
if candidate is None and not force:
candidate = click.prompt('Candidate ' + str(candidate_names), type=str, default='')
if action == 'failover' and not candidate:
raise PatroniCtlException('Failover could be performed only to a specific candidate')
if candidate == master:
raise PatroniCtlException(action.title() + ' target and source are the same.')
raise PatroniCtlException('Failover target and source are the same.')
if candidate and candidate not in candidate_names:
raise PatroniCtlException('Member {0} does not exist in cluster {1}'.format(candidate, cluster_name))
if scheduled is None and not force:
scheduled = click.prompt('When should the failover take place (e.g. 2015-10-01T14:30) ', type=str,
default='now')
scheduled_at = parse_scheduled(scheduled)
scheduled_at_str = None
if action == 'switchover':
if scheduled is None and not force:
scheduled = click.prompt('When should the switchover take place (e.g. 2015-10-01T14:30) ',
type=str, default='now')
scheduled_at = parse_scheduled(scheduled)
if scheduled_at:
if cluster.is_paused():
raise PatroniCtlException("Can't schedule switchover in the paused state")
scheduled_at_str = scheduled_at.isoformat()
if scheduled_at:
if cluster.is_paused():
raise PatroniCtlException("Can't schedule failover in the paused state")
scheduled_at_str = scheduled_at.isoformat()
failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at_str}
@@ -646,80 +588,46 @@ def _do_failover_or_switchover(obj, action, cluster_name, master, candidate, for
output_members(dcs.get_cluster(), cluster_name)
if not force:
demote_msg = ', demoting current master ' + master if master else ''
if not click.confirm('Are you sure you want to {0} cluster {1}{2}?'.format(action, cluster_name, demote_msg)):
raise PatroniCtlException('Aborting ' + action)
a = \
click.confirm('Are you sure you want to failover cluster {0}, demoting current master {1}?'.format(
cluster_name, master))
if not a:
raise PatroniCtlException('Aborting failover')
r = None
try:
member = cluster.leader.member if cluster.leader else cluster.get_member(candidate, False)
r = request_patroni(member, 'post', action, failover_value, auth_header(obj))
# probably old patroni, which doesn't support switchover yet
if r.status_code == 501 and action == 'switchover' and 'Server does not support this operation' in r.text:
r = request_patroni(member, 'post', 'failover', failover_value, auth_header(obj))
member = cluster.leader.member if cluster.leader else [m for m in cluster.members if m.name == candidate][0]
r = request_patroni(member, 'post', 'failover', failover_value, auth_header(obj))
if r.status_code in (200, 202):
logging.debug(r)
cluster = dcs.get_cluster()
logging.debug(cluster)
click.echo('{0} {1}'.format(timestamp(), r.text))
else:
click.echo('{0} failed, details: {1}, {2}'.format(action.title(), r.status_code, r.text))
click.echo('Failover failed, details: {0}, {1}'.format(r.status_code, r.text))
return
except Exception:
logging.exception(r)
logging.warning('Failing over to DCS')
click.echo('{0} Could not {1} using Patroni api, falling back to DCS'.format(timestamp(), action))
click.echo(timestamp() + ' Could not failover using Patroni api, falling back to DCS')
click.echo(timestamp() + ' Initializing failover from master {0}'.format(master))
dcs.manual_failover(master, candidate, scheduled_at=scheduled_at)
output_members(cluster, cluster_name)
@ctl.command('failover', help='Failover to a replica')
@arg_cluster_name
@click.option('--master', help='The name of the current master', default=None)
@click.option('--candidate', help='The name of the candidate', default=None)
@option_force
@click.pass_obj
def failover(obj, cluster_name, master, candidate, force):
action = 'switchover' if master else 'failover'
_do_failover_or_switchover(obj, action, cluster_name, master, candidate, force)
@ctl.command('switchover', help='Switchover to a replica')
@arg_cluster_name
@click.option('--master', help='The name of the current master', default=None)
@click.option('--candidate', help='The name of the candidate', default=None)
@click.option('--scheduled', help='Timestamp of a scheduled switchover in unambiguous format (e.g. ISO 8601)',
default=None)
@option_force
@click.pass_obj
def switchover(obj, cluster_name, master, candidate, force, scheduled):
_do_failover_or_switchover(obj, 'switchover', cluster_name, master, candidate, force, scheduled)
def output_members(cluster, name, extended=False, fmt='pretty'):
rows = []
logging.debug(cluster)
leader_name = None
if cluster.leader:
leader_name = cluster.leader.name
leader_name = cluster.leader.member.name
xlog_location_cluster = cluster.last_leader_operation or 0
# Mainly for consistent pretty printing and watching we sort the output
cluster.members.sort(key=lambda x: x.name)
has_scheduled_restarts = any(m.data.get('scheduled_restart') for m in cluster.members)
has_pending_restarts = any(m.data.get('pending_restart') for m in cluster.members)
# Show Host as 'host:port' if somebody is running on non-standard port or two nodes are running on the same host
append_port = any(str(m.conn_kwargs()['port']) != '5432' for m in cluster.members) or\
len(set(m.conn_kwargs()['host'] for m in cluster.members)) < len(cluster.members)
for m in cluster.members:
logging.debug(m)
@@ -729,23 +637,22 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
elif m.name == cluster.sync.sync_standby:
role = 'Sync standby'
xlog_location = m.data.get('xlog_location')
host = m.conn_kwargs()['host']
xlog_location = m.data.get('xlog_location') or 0
lag = ''
if xlog_location is None:
lag = 'unknown'
elif xlog_location_cluster >= xlog_location:
if xlog_location_cluster >= xlog_location:
lag = round((xlog_location_cluster - xlog_location)/1024/1024)
host = m.conn_kwargs()['host']
if append_port:
host += ':{0}'.format(m.conn_kwargs()['port'])
row = [name, m.name, host, role, m.data.get('state', ''), m.data.get('timeline', ''), lag]
if extended or has_pending_restarts:
row.append('*' if m.data.get('pending_restart') else '')
if extended or has_scheduled_restarts:
row = [
name,
m.name,
host,
role,
m.data.get('state', ''),
lag,
]
if extended:
value = ''
scheduled_restart = m.data.get('scheduled_restart')
if scheduled_restart:
@@ -757,55 +664,39 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
rows.append(row)
columns = ['Cluster', 'Member', 'Host', 'Role', 'State', 'TL', 'Lag in MB']
alignment = {'Lag in MB': 'r', 'TL': 'r'}
columns = [
'Cluster',
'Member',
'Host',
'Role',
'State',
'Lag in MB',
]
alignment = {'Cluster': 'l', 'Member': 'l', 'Host': 'l', 'Lag in MB': 'r'}
if extended or has_pending_restarts:
columns.append('Pending restart')
if extended or has_scheduled_restarts:
if extended:
columns.append('Scheduled restart')
alignment['Scheduled restart'] = 'l'
print_output(columns, rows, alignment, fmt)
service_info = []
if cluster.is_paused():
service_info.append('Maintenance mode: on')
if cluster.failover and cluster.failover.scheduled_at:
info = 'Switchover scheduled at: ' + cluster.failover.scheduled_at.isoformat()
if cluster.failover.leader:
info += '\n from: ' + cluster.failover.leader
if cluster.failover.candidate:
info += '\n to: ' + cluster.failover.candidate
service_info.append(info)
if service_info:
click.echo(' ' + '\n '.join(service_info))
@ctl.command('list', help='List the Patroni members for a given Patroni')
@click.argument('cluster_names', nargs=-1)
@click.option('--extended', '-e', help='Show some extra information', is_flag=True)
@click.option('--timestamp', '-t', 'ts', help='Print timestamp', is_flag=True)
@option_format
@option_watch
@option_watchrefresh
@click.pass_obj
def members(obj, cluster_names, fmt, watch, w, extended, ts):
def members(obj, cluster_names, fmt, watch, w, extended):
if not cluster_names:
if 'scope' in obj:
cluster_names = [obj['scope']]
if not cluster_names:
return logging.warning('Listing members: No cluster names were provided')
logging.warning('Listing members: No cluster names were provided')
return
for cluster_name in cluster_names:
dcs = get_dcs(obj, cluster_name)
for _ in watching(w, watch):
if ts:
click.echo(timestamp(0))
cluster = dcs.get_cluster()
output_members(cluster, cluster_name, extended, fmt)
@@ -841,7 +732,7 @@ def touch_member(config, dcs):
'role': p.role
}
return dcs.touch_member(data, permanent=True)
return dcs.touch_member(json.dumps(data, separators=(',', ':')), permanent=True)
def set_defaults(config, cluster_name):
@@ -898,27 +789,8 @@ def flush(obj, cluster_name, member_names, force, role, target):
click.echo('No scheduled restart for member {0}'.format(member.name))
def wait_until_pause_is_applied(dcs, paused, old_cluster):
click.echo("'{0}' request sent, waiting until it is recognized by all nodes".format(paused and 'pause' or 'resume'))
old = {m.name: m.index for m in old_cluster.members if m.api_url}
loop_wait = old_cluster.config.data.get('loop_wait', dcs.loop_wait)
for _ in polling_loop(loop_wait + 1):
cluster = dcs.get_cluster()
if all(m.data.get('pause', False) == paused for m in cluster.members if m.name in old):
break
else:
remaining = [m.name for m in cluster.members if m.data.get('pause', False) != paused
and m.name in old and old[m.name] != m.index]
if remaining:
return click.echo("{0} members didn't recognized pause state after {1} seconds"
.format(', '.join(remaining), loop_wait))
return click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
def toggle_pause(config, cluster_name, paused, wait):
dcs = get_dcs(config, cluster_name)
cluster = dcs.get_cluster()
def toggle_pause(config, cluster_name, paused):
cluster = get_dcs(config, cluster_name).get_cluster()
if cluster.is_paused() == paused:
raise PatroniCtlException('Cluster is {0} paused'.format(paused and 'already' or 'not'))
@@ -930,16 +802,12 @@ def toggle_pause(config, cluster_name, paused, wait):
for member in members:
try:
r = request_patroni(member, 'patch', 'config', {'pause': paused or None}, auth_header(config))
except Exception as err:
logging.warning(str(err))
except Exception:
logging.warning('Member %s is not accessible', member.name)
continue
if r.status_code == 200:
if wait:
wait_until_pause_is_applied(dcs, paused, cluster)
else:
click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
else:
click.echo('Failed: {0} cluster management status code={1}, ({2})'.format(
paused and 'pause' or 'resume', r.status_code, r.text))
@@ -949,19 +817,17 @@ def toggle_pause(config, cluster_name, paused, wait):
@ctl.command('pause', help='Disable auto failover')
@arg_cluster_name
@click.argument('cluster_name')
@click.pass_obj
@click.option('--wait', help='Wait until pause is applied on all nodes', is_flag=True)
def pause(obj, cluster_name, wait):
return toggle_pause(obj, cluster_name, True, wait)
def pause(obj, cluster_name):
return toggle_pause(obj, cluster_name, True)
@ctl.command('resume', help='Resume auto failover')
@arg_cluster_name
@click.option('--wait', help='Wait until pause is cleared on all nodes', is_flag=True)
@click.argument('cluster_name')
@click.pass_obj
def resume(obj, cluster_name, wait):
return toggle_pause(obj, cluster_name, False, wait)
def resume(obj, cluster_name):
return toggle_pause(obj, cluster_name, False)
@contextmanager
@@ -1037,14 +903,14 @@ def apply_config_changes(before_editing, data, kvpairs):
if prefix == ('postgresql', 'parameters'):
path = ['.'.join(path)]
key = path[0]
if len(path) == 1:
if value is None:
config.pop(key, None)
config.pop(path[0], None)
else:
config[key] = value
config[path[0]] = value
else:
if not isinstance(config.get(key), dict):
key = path[0]
if key not in config:
config[key] = {}
set_path_value(config[key], path[1:], value, prefix + (key,))
if config[key] == {}:
@@ -1103,7 +969,7 @@ def invoke_editor(before_editing, cluster_name):
@ctl.command('edit-config', help="Edit cluster configuration")
@arg_cluster_name
@click.argument('cluster_name')
@click.option('--quiet', '-q', is_flag=True, help='Do not show changes')
@click.option('--set', '-s', 'kvpairs', multiple=True,
help='Set specific configuration value. Can be specified multiple times')
@@ -1151,48 +1017,15 @@ def edit_config(obj, cluster_name, force, quiet, kvpairs, pgkvpairs, apply_filen
return
if force or click.confirm('Apply these changes?'):
if not dcs.set_config_value(json.dumps(changed_data), cluster.config.index):
if not dcs.set_config_value(json.dumps(changed_data), cluster.config.modify_index):
raise PatroniCtlException("Config modification aborted due to concurrent changes")
click.echo("Configuration changed")
@ctl.command('show-config', help="Show cluster configuration")
@arg_cluster_name
@click.argument('cluster_name')
@click.pass_obj
def show_config(obj, cluster_name):
cluster = get_dcs(obj, cluster_name).get_cluster()
click.echo(format_config_for_editing(cluster.config.data))
@ctl.command('version', help='Output version of patronictl command or a running Patroni instance')
@click.argument('cluster_name', required=False)
@click.argument('member_names', nargs=-1)
@click.pass_obj
def version(obj, cluster_name, member_names):
click.echo("patronictl version {0}".format(__version__))
if not cluster_name:
return
click.echo("")
cluster = get_dcs(obj, cluster_name).get_cluster()
for m in cluster.members:
if m.api_url:
if not member_names or m.name in member_names:
try:
response = request_patroni(m, 'get', 'patroni')
data = response.json()
version = data.get('patroni', {}).get('version')
pg_version = data.get('server_version')
pg_version_str = " PostgreSQL {0}".format(format_pg_version(pg_version)) if pg_version else ""
click.echo("{0}: Patroni {1}{2}".format(m.name, version, pg_version_str))
except Exception as e:
click.echo("{0}: failed to get version: {1}".format(m.name, e))
def format_pg_version(version):
if version < 100000:
return "{0}.{1}.{2}".format(version // 10000, version // 100 % 100, version % 100)
else:
return "{0}.{1}".format(version // 10000, version % 100)
+31 -240
View File
@@ -6,38 +6,18 @@ import json
import logging
import os
import pkgutil
import re
import six
import sys
from collections import defaultdict, namedtuple
from copy import deepcopy
from collections import namedtuple
from patroni.exceptions import PatroniException
from patroni.utils import parse_bool
from random import randint
from six.moves.urllib_parse import urlparse, urlunparse, parse_qsl
from threading import Event, Lock
slot_name_re = re.compile('^[a-z0-9_]{1,63}$')
logger = logging.getLogger(__name__)
def slot_name_from_member_name(member_name):
"""Translate member name to valid PostgreSQL slot name.
PostgreSQL replication slot names must be valid PostgreSQL names. This function maps the wider space of
member names to valid PostgreSQL names. Names are lowercased, dashes and periods common in hostnames
are replaced with underscores, other characters are encoded as their unicode codepoint. Name is truncated
to 64 characters. Multiple different member names may map to a single slot name."""
def replace_char(match):
c = match.group(0)
return '_' if c in '-.' else "u{:04d}".format(ord(c))
slot_name = re.sub('[^a-z0-9_]', replace_char, member_name.lower())
return slot_name[0:63]
def parse_connection_string(value):
"""Original Governor stores connection strings for each cluster members if a following format:
postgres://{username}:{password}@{connect_address}/postgres
@@ -127,29 +107,12 @@ class Member(namedtuple('Member', 'index,name,session,data')):
@property
def conn_url(self):
conn_url = self.data.get('conn_url')
conn_kwargs = self.data.get('conn_kwargs')
if conn_url:
return conn_url
if conn_kwargs:
conn_url = 'postgresql://{host}:{port}'.format(
host=conn_kwargs.get('host'),
port=conn_kwargs.get('port'),
)
self.data['conn_url'] = conn_url
return conn_url
return self.data.get('conn_url')
def conn_kwargs(self, auth=None):
defaults = {
"host": "",
"port": "",
"database": ""
}
ret = self.data.get('conn_kwargs')
if ret:
defaults.update(ret)
ret = defaults
ret = ret.copy()
else:
r = urlparse(self.conn_url)
ret = {
@@ -195,26 +158,6 @@ class Member(namedtuple('Member', 'index,name,session,data')):
return self.state == 'running'
class RemoteMember(Member):
""" Represents a remote master for a standby cluster
"""
def __new__(cls, name, data):
return super(RemoteMember, cls).__new__(cls, None, name, None, data)
@staticmethod
def allowed_keys():
return ('primary_slot_name',
'create_replica_methods',
'restore_command',
'archive_cleanup_command',
'recovery_min_apply_delay',
'no_replication_slot')
def __getattr__(self, name):
if name in RemoteMember.allowed_keys():
return self.data.get(name)
class Leader(namedtuple('Leader', 'index,session,member')):
"""Immutable object (namedtuple) which represents leader key.
@@ -234,48 +177,40 @@ class Leader(namedtuple('Leader', 'index,session,member')):
def conn_url(self):
return self.member.conn_url
@property
def timeline(self):
return self.member.data.get('timeline')
class Failover(namedtuple('Failover', 'index,leader,candidate,scheduled_at')):
"""
>>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader"}'))
True
>>> 'Failover' in str(Failover.from_node(1, {"leader": "cluster_leader"}))
True
>>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader", "member": "cluster_candidate"}'))
True
>>> Failover.from_node(1, 'null') is None
False
True
>>> n = '{"leader": "cluster_leader", "member": "cluster_candidate", "scheduled_at": "2016-01-14T10:09:57.1394Z"}'
>>> 'tzinfo=' in str(Failover.from_node(1, n))
True
>>> Failover.from_node(1, None) is None
False
True
>>> Failover.from_node(1, '{}') is None
False
True
>>> 'abc' in Failover.from_node(1, 'abc:def')
True
"""
@staticmethod
def from_node(index, value):
if isinstance(value, dict):
data = value
elif value:
try:
data = json.loads(value)
if not isinstance(data, dict):
data = {}
except ValueError:
t = [a.strip() for a in value.split(':')]
leader = t[0]
candidate = t[1] if len(t) > 1 else None
return Failover(index, leader, candidate, None) if leader or candidate else None
else:
data = {}
if not value:
return None
try:
data = json.loads(value)
if not data:
return None
except ValueError:
t = [a.strip() for a in value.split(':')]
leader = t[0]
candidate = t[1] if len(t) > 1 else None
return Failover(index, leader, candidate, None) if leader or candidate else None
if data.get('scheduled_at'):
data['scheduled_at'] = dateutil.parser.parse(data['scheduled_at'])
@@ -292,24 +227,14 @@ class ClusterConfig(namedtuple('ClusterConfig', 'index,data,modify_index')):
def from_node(index, data, modify_index=None):
"""
>>> ClusterConfig.from_node(1, '{') is None
False
True
"""
try:
data = json.loads(data)
except (TypeError, ValueError):
data = None
modify_index = 0
if not isinstance(data, dict):
data = {}
return ClusterConfig(index, data, index if modify_index is None else modify_index)
@property
def permanent_slots(self):
return isinstance(self.data, dict) and (
self.data.get('permanent_replication_slots') or
self.data.get('permanent_slots') or self.data.get('slots')
) or {}
return None
return ClusterConfig(index, data, modify_index or index)
class SyncState(namedtuple('SyncState', 'index,leader,sync_standby')):
@@ -333,12 +258,8 @@ class SyncState(namedtuple('SyncState', 'index,leader,sync_standby')):
True
>>> SyncState.from_node(1, '{"leader": "leader"}').leader == "leader"
True
>>> SyncState.from_node(1, {"leader": "leader"}).leader == "leader"
True
"""
if isinstance(value, dict):
data = value
elif value:
if value:
try:
data = json.loads(value)
if not isinstance(data, dict):
@@ -368,26 +289,7 @@ class SyncState(namedtuple('SyncState', 'index,leader,sync_standby')):
return name is not None and name in (self.leader, self.sync_standby)
class TimelineHistory(namedtuple('TimelineHistory', 'index,value,lines')):
"""Object representing timeline history file"""
@staticmethod
def from_node(index, value):
"""
>>> h = TimelineHistory.from_node(1, 2)
>>> h.lines
[]
"""
try:
lines = json.loads(value)
except (TypeError, ValueError):
lines = None
if not isinstance(lines, list):
lines = []
return TimelineHistory(index, value, lines)
class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operation,members,failover,sync,history')):
class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operation,members,failover,sync')):
"""Immutable object (namedtuple) which represents PostgreSQL cluster.
Consists of the following fields:
@@ -399,7 +301,6 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operat
:param members: list of Member object, all PostgreSQL cluster members including leader
:param failover: reference to `Failover` object
:param sync: reference to `SyncState` object, last observed synchronous replication state.
:param history: reference to `TimelineHistory` object
"""
def is_unlocked(self):
@@ -416,93 +317,8 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operat
candidates = [m for m in self.members if m.clonefrom and m.is_running and m.name not in exclude]
return candidates[randint(0, len(candidates) - 1)] if candidates else self.leader
def check_mode(self, mode):
return bool(self.config and parse_bool(self.config.data.get(mode)))
def is_paused(self):
return self.check_mode('pause')
def is_synchronous_mode(self):
return self.check_mode('synchronous_mode')
def get_replication_slots(self, name, role):
# if the replicatefrom tag is set on the member - we should not create the replication slot for it on
# the current master, because that member would replicate from elsewhere. We still create the slot if
# the replicatefrom destination member is currently not a member of the cluster (fallback to the
# master), or if replicatefrom destination member happens to be the current master
if role in ('master', 'standby_leader'):
slot_members = [m.name for m in self.members if m.name != name and
(m.replicatefrom is None or m.replicatefrom == name or
not self.has_member(m.replicatefrom))]
permanent_slots = (self.config and self.config.permanent_slots or {}).copy()
else:
# only manage slots for replicas that replicate from this one, except for the leader among them
slot_members = [m.name for m in self.members if m.replicatefrom == name and m.name != self.leader.name]
permanent_slots = {}
slots = {slot_name_from_member_name(name): {'type': 'physical'} for name in slot_members}
if len(slots) < len(slot_members):
# Find which names are conflicting for a nicer error message
slot_conflicts = defaultdict(list)
for name in slot_members:
slot_conflicts[slot_name_from_member_name(name)].append(name)
logger.error("Following cluster members share a replication slot name: %s",
"; ".join("{} map to {}".format(", ".join(v), k)
for k, v in slot_conflicts.items() if len(v) > 1))
# "merge" replication slots for members with permanent_replication_slots
for name, value in permanent_slots.items():
if not slot_name_re.match(name):
logger.error("Invalid permanent replication slot name '%s'", name)
logger.error("Slot name may only contain lower case letters, numbers, and the underscore chars")
continue
if name in slots:
logger.error("Permanent replication slot {'%s': %s} is conflicting with" +
" physical replication slot for cluster member", name, value)
continue
value = deepcopy(value)
if not value:
value = {'type': 'physical'}
if isinstance(value, dict):
if 'type' not in value:
value['type'] = 'logical' if value.get('database') and value.get('plugin') else 'physical'
if value['type'] == 'physical' or value['type'] == 'logical' \
and value.get('database') and value.get('plugin'):
slots[name] = value
continue
logger.error("Bad value for slot '%s' in permanent_slots: %s", name, permanent_slots[name])
return slots
def has_permanent_logical_slots(self, name):
slots = self.get_replication_slots(name, 'master').values()
return any(v for v in slots if v.get("type") == "logical")
@property
def timeline(self):
"""
>>> Cluster(0, 0, 0, 0, 0, 0, 0, 0).timeline
0
>>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[]')).timeline
1
>>> Cluster(0, 0, 0, 0, 0, 0, 0, TimelineHistory.from_node(1, '[["a"]]')).timeline
0
"""
if self.history:
if self.history.lines:
try:
return int(self.history.lines[-1][0]) + 1
except Exception:
logger.error('Failed to parse cluster history from DCS: %s', self.history.lines)
elif self.history.value == '[]':
return 1
return 0
return self.config and self.config.data.get('pause', False) or False
@six.add_metaclass(abc.ABCMeta)
@@ -512,7 +328,6 @@ class AbstractDCS(object):
_CONFIG = 'config'
_LEADER = 'leader'
_FAILOVER = 'failover'
_HISTORY = 'history'
_MEMBERS = 'members/'
_OPTIME = 'optime'
_LEADER_OPTIME = _OPTIME + '/' + _LEADER
@@ -524,7 +339,8 @@ class AbstractDCS(object):
i.e.: `zookeeper` for zookeeper, `etcd` for etcd, etc...
"""
self._name = config['name']
self._base_path = re.sub('/+', '/', '/'.join(['', config.get('namespace', 'service'), config['scope']]))
self._namespace = '/{0}'.format(config.get('namespace', '/service/').strip('/'))
self._base_path = '/'.join([self._namespace, config['scope']])
self._set_loop_wait(config.get('loop_wait', 10))
self._ctl = bool(config.get('patronictl', False))
@@ -560,10 +376,6 @@ class AbstractDCS(object):
def failover_path(self):
return self.client_path(self._FAILOVER)
@property
def history_path(self):
return self.client_path(self._HISTORY)
@property
def leader_optime_path(self):
return self.client_path(self._LEADER_OPTIME)
@@ -606,7 +418,7 @@ class AbstractDCS(object):
with self._cluster_thread_lock:
try:
self._load_cluster()
except Exception:
except:
self._cluster = None
raise
return self._cluster
@@ -631,7 +443,7 @@ class AbstractDCS(object):
self._last_leader_operation = last_operation
@abc.abstractmethod
def _update_leader(self):
def update_leader(self):
"""Update leader key (or session) ttl
:returns: `!True` if leader key (or session) has been updated successfully.
@@ -640,18 +452,6 @@ class AbstractDCS(object):
You have to use CAS (Compare And Swap) operation in order to update leader key,
for example for etcd `prevValue` parameter must be used."""
def update_leader(self, last_operation, access_is_restricted=False):
"""Update leader key (or session) ttl and optime/leader
:param last_operation: absolute xlog location in bytes
:returns: `!True` if leader key (or session) has been updated successfully.
If not, `!False` must be returned and current instance would be demoted."""
ret = self._update_leader()
if ret and last_operation:
self.write_leader_optime(last_operation)
return ret
@abc.abstractmethod
def attempt_to_acquire_leader(self, permanent=False):
"""Attempt to acquire leader lock
@@ -677,6 +477,7 @@ class AbstractDCS(object):
if scheduled_at:
failover_value['scheduled_at'] = scheduled_at.isoformat()
return self.set_failover_value(json.dumps(failover_value, separators=(',', ':')), index)
@abc.abstractmethod
@@ -689,7 +490,7 @@ class AbstractDCS(object):
This method should create or update key with the name = '/members/' + `~self._name`
and value = data in a given DCS.
:param data: information about instance (including connection strings)
:param data: json serialized information about instance (including connection strings)
:param ttl: ttl for member key, optional parameter. If it is None `~self.member_ttl will be used`
:param permanent: if set to `!True`, the member key will never expire.
Used in patronictl for the external master.
@@ -726,18 +527,8 @@ class AbstractDCS(object):
def delete_cluster(self):
"""Delete cluster from DCS"""
@staticmethod
def sync_state(leader, sync_standby):
"""Build sync_state dict"""
return {'leader': leader, 'sync_standby': sync_standby}
def write_sync_state(self, leader, sync_standby, index=None):
sync_value = self.sync_state(leader, sync_standby)
return self.set_sync_state_value(json.dumps(sync_value, separators=(',', ':')), index)
@abc.abstractmethod
def set_history_value(self, value):
""""""
return self.set_sync_state_value(json.dumps({'leader': leader, 'sync_standby': sync_standby}), index=index)
@abc.abstractmethod
def set_sync_state_value(self, value, index=None):
+42 -241
View File
@@ -1,19 +1,16 @@
from __future__ import absolute_import
import json
import logging
import os
import re
import socket
import ssl
import time
import urllib3
from consul import ConsulException, NotFound, base
from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
from patroni.exceptions import DCSError
from patroni.utils import deep_compare, parse_bool, Retry, RetryFailedError, split_host_port
from patroni.utils import Retry, RetryFailedError
from urllib3.exceptions import HTTPError
from six.moves.urllib.parse import urlencode, urlparse, quote
from six.moves.urllib.parse import urlencode
from six.moves.http_client import HTTPException
logger = logging.getLogger(__name__)
@@ -27,43 +24,21 @@ class ConsulInternalError(ConsulException):
"""An internal Consul server error occurred"""
class InvalidSessionTTL(ConsulException):
"""Session TTL is too small or too big"""
class InvalidSession(ConsulException):
"""invalid session"""
class HTTPClient(object):
def __init__(self, host='127.0.0.1', port=8500, token=None, scheme='http', verify=True, cert=None, ca_cert=None):
self.token = token
self._read_timeout = 10
self.base_uri = '{0}://{1}:{2}'.format(scheme, host, port)
kwargs = {}
if cert:
if isinstance(cert, tuple):
# Key and cert are separate
kwargs['cert_file'] = cert[0]
kwargs['key_file'] = cert[1]
else:
# combined certificate
kwargs['cert_file'] = cert
if ca_cert:
kwargs['ca_certs'] = ca_cert
if verify or ca_cert:
kwargs['cert_reqs'] = ssl.CERT_REQUIRED
self.http = urllib3.PoolManager(num_pools=10, **kwargs)
def __init__(self, host='127.0.0.1', port=8500, scheme='http', verify=True, timeout=10):
self.host = host
self.port = port
self.scheme = scheme
self.verify = verify
self.set_read_timeout(timeout)
self.base_uri = '{0}://{1}:{2}'.format(self.scheme, self.host, self.port)
self.http = urllib3.PoolManager(num_pools=10)
self._ttl = None
def set_read_timeout(self, timeout):
self._read_timeout = timeout/3.0
@property
def ttl(self):
return self._ttl
def set_ttl(self, ttl):
ret = self._ttl != ttl
self._ttl = ttl
@@ -73,13 +48,7 @@ class HTTPClient(object):
def response(response):
data = response.data.decode('utf-8')
if response.status == 500:
msg = '{0} {1}'.format(response.status, data)
if data.startswith('Invalid Session TTL'):
raise InvalidSessionTTL(msg)
elif data.startswith('invalid session'):
raise InvalidSession(msg)
else:
raise ConsulInternalError(msg)
raise ConsulInternalError('{0} {1}'.format(response.status, data))
return base.Response(response.status, response.headers, data)
def uri(self, path, params=None):
@@ -98,42 +67,20 @@ class HTTPClient(object):
data = '{' + ttl + '}'
else:
data = data[:-1] + ', ' + ttl + '}'
if isinstance(params, list): # starting from v1.1.0 python-consul switched from `dict` to `list` for params
params = {k: v for k, v in params}
kwargs = {'retries': 0, 'preload_content': False, 'body': data}
if method == 'get' and isinstance(params, dict) and 'index' in params:
timeout = float(params['wait'][:-1]) if 'wait' in params else 300
# According to the documentation a small random amount of additional wait time is added to the
# supplied maximum wait time to spread out the wake up time of any concurrent requests. This adds
# up to wait / 16 additional time to the maximum duration. Since our goal is actually getting a
# response rather read timeout we will add to the timeout a sligtly bigger value.
kwargs['timeout'] = timeout + max(timeout/15.0, 1)
kwargs['timeout'] = (float(params['wait'][:-1]) if 'wait' in params else 300) + 1
else:
kwargs['timeout'] = self._read_timeout
token = params.pop('token', self.token) if isinstance(params, dict) else self.token
if token:
kwargs['headers'] = {'X-Consul-Token': token}
return callback(self.response(self.http.request(method.upper(), self.uri(path, params), **kwargs)))
return wrapper
class ConsulClient(base.Consul):
def __init__(self, *args, **kwargs):
self._cert = kwargs.pop('cert', None)
self._ca_cert = kwargs.pop('ca_cert', None)
self._token = kwargs.get('token')
super(ConsulClient, self).__init__(*args, **kwargs)
def connect(self, *args, **kwargs):
kwargs.update(dict(zip(['host', 'port', 'scheme', 'verify'], args)))
if self._cert:
kwargs['cert'] = self._cert
if self._ca_cert:
kwargs['ca_cert'] = self._ca_cert
if self._token:
kwargs['token'] = self._token
return HTTPClient(**kwargs)
@staticmethod
def connect(host, port, scheme, verify=True):
return HTTPClient(host, port, scheme, verify)
def catch_consul_errors(func):
@@ -145,31 +92,6 @@ def catch_consul_errors(func):
return wrapper
def force_if_last_failed(func):
def wrapper(*args, **kwargs):
if wrapper.last_result is False:
kwargs['force'] = True
wrapper.last_result = func(*args, **kwargs)
return wrapper.last_result
wrapper.last_result = None
return wrapper
def service_name_from_scope_name(scope_name):
"""Translate scope name to service name which can be used in dns.
230 = 253 - len('replica.') - len('.service.consul')
"""
def replace_char(match):
c = match.group(0)
return '-' if c in '. _' else "u{:04d}".format(ord(c))
service_name = re.sub(r'[^a-z0-9\-]', replace_char, scope_name.lower())
return service_name[0:230]
class Consul(AbstractDCS):
def __init__(self, config):
@@ -181,43 +103,12 @@ class Consul(AbstractDCS):
retry_exceptions=(ConsulInternalError, HTTPException,
HTTPError, socket.error, socket.timeout))
kwargs = {}
if 'url' in config:
r = urlparse(config['url'])
config.update({'scheme': r.scheme, 'host': r.hostname, 'port': r.port or 8500})
elif 'host' in config:
host, port = split_host_port(config.get('host', '127.0.0.1:8500'), 8500)
config['host'] = host
if 'port' not in config:
config['port'] = int(port)
if config.get('cacert'):
config['ca_cert'] = config.pop('cacert')
if config.get('key') and config.get('cert'):
config['cert'] = (config['cert'], config['key'])
config_keys = ('host', 'port', 'token', 'scheme', 'cert', 'ca_cert', 'dc')
kwargs = {p: config.get(p) for p in config_keys if config.get(p)}
verify = config.get('verify')
if not isinstance(verify, bool):
verify = parse_bool(verify)
if isinstance(verify, bool):
kwargs['verify'] = verify
self._client = ConsulClient(**kwargs)
self._my_member_data = None
host, port = config.get('host', '127.0.0.1:8500').split(':')
self._client = ConsulClient(host=host, port=port)
self.set_retry_timeout(config['retry_timeout'])
self.set_ttl(config.get('ttl') or 30)
self._last_session_refresh = 0
self.__session_checks = config.get('checks')
self._register_service = config.get('register_service', False)
if self._register_service:
self._service_name = service_name_from_scope_name(self._scope)
if self._scope != self._service_name:
logger.warning('Using %s as consul service name instead of scope name %s', self._service_name,
self._scope)
self._service_check_interval = config.get('service_check_interval', '5s')
if not self._ctl:
self.create_session()
@@ -241,15 +132,6 @@ class Consul(AbstractDCS):
self._retry.deadline = retry_timeout
self._client.http.set_read_timeout(retry_timeout)
def adjust_ttl(self):
try:
settings = self._client.agent.self()
min_ttl = (settings['Config']['SessionTTLMin'] or 10000000000)/1000000000.0
logger.warning('Changing Session TTL from %s to %s', self._client.http.ttl, min_ttl)
self._client.http.set_ttl(min_ttl)
except Exception:
logger.exception('adjust_ttl')
def _do_refresh_session(self):
""":returns: `!True` if it had to create new session"""
if self._session and self._last_session_refresh + self._loop_wait > time.time():
@@ -262,15 +144,8 @@ class Consul(AbstractDCS):
self._session = None
ret = not self._session
if ret:
try:
self._session = self._client.session.create(name=self._scope + '-' + self._name,
checks=self.__session_checks,
lock_delay=0.001, behavior='delete')
except InvalidSessionTTL:
logger.exception('session.create')
self.adjust_ttl()
raise
self._session = self._client.session.create(name=self._scope + '-' + self._name,
lock_delay=0.001, behavior='delete')
self._last_session_refresh = time.time()
return ret
@@ -299,7 +174,7 @@ class Consul(AbstractDCS):
nodes = {}
for node in results:
node['Value'] = (node['Value'] or b'').decode('utf-8')
nodes[os.path.relpath(node['Key'], path).replace('\\', '/')] = node
nodes[os.path.relpath(node['Key'], path)] = node
# get initialize flag
initialize = nodes.get(self._INITIALIZE)
@@ -309,10 +184,6 @@ class Consul(AbstractDCS):
config = nodes.get(self._CONFIG)
config = config and ClusterConfig.from_node(config['ModifyIndex'], config['Value'])
# get timeline history
history = nodes.get(self._HISTORY)
history = history and TimelineHistory.from_node(history['ModifyIndex'], history['Value'])
# get last leader operation
last_leader_operation = nodes.get(self._LEADER_OPTIME)
last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation['Value'])
@@ -342,111 +213,46 @@ class Consul(AbstractDCS):
sync = nodes.get(self._SYNC)
sync = SyncState.from_node(sync and sync['ModifyIndex'], sync and sync['Value'])
self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
except NotFound:
self._cluster = Cluster(None, None, None, None, [], None, None, None)
except Exception:
self._cluster = Cluster(None, None, None, None, [], None, None)
except:
logger.exception('get_cluster')
raise ConsulError('Consul is not responding properly')
@catch_consul_errors
def touch_member(self, data, ttl=None, permanent=False):
def touch_member(self, data, **kwargs):
cluster = self.cluster
member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
create_member = not permanent and self.refresh_session()
create_member = self.refresh_session()
if member and (create_member or member.session != self._session):
self._client.kv.delete(self.member_path)
create_member = True
try:
self._client.kv.delete(self.member_path)
create_member = True
except Exception:
return False
if not create_member and member and deep_compare(data, member.data):
if not create_member and member and data == self._my_member_data:
return True
try:
args = {} if permanent else {'acquire': self._session}
self._client.kv.put(self.member_path, json.dumps(data, separators=(',', ':')), **args)
if self._register_service:
self.update_service(not create_member and member and member.data or {}, data)
args = {} if kwargs.get('permanent', False) else {'acquire': self._session}
self._client.kv.put(self.member_path, data, **args)
self._my_member_data = data
return True
except InvalidSession:
self._session = None
logger.error('Our session disappeared from Consul, can not "touch_member"')
except Exception:
logger.exception('touch_member')
return False
@catch_consul_errors
def register_service(self, service_name, **kwargs):
logger.info('Register service %s, params %s', service_name, kwargs)
return self._client.agent.service.register(service_name, **kwargs)
@catch_consul_errors
def deregister_service(self, service_id):
logger.info('Deregister service %s', service_id)
# service_id can contain special characters, but is used as part of uri in deregister request
service_id = quote(service_id)
return self._client.agent.service.deregister(service_id)
def _update_service(self, data):
service_name = self._service_name
role = data['role'].replace('_', '-')
state = data['state']
api_parts = urlparse(data['api_url'])
api_parts = api_parts._replace(path='/{0}'.format(role))
conn_parts = urlparse(data['conn_url'])
check = base.Check.http(api_parts.geturl(), self._service_check_interval,
deregister='{0}s'.format(self._client.http.ttl * 10))
params = {
'service_id': '{0}/{1}'.format(self._scope, self._name),
'address': conn_parts.hostname,
'port': conn_parts.port,
'check': check,
'tags': [role]
}
if state == 'stopped':
return self.deregister_service(params['service_id'])
if role in ['master', 'replica', 'standby-leader']:
if state != 'running':
return
return self.register_service(service_name, **params)
logger.warning('Could not register service: unknown role type %s', role)
@force_if_last_failed
def update_service(self, old_data, new_data, force=False):
update = False
for key in ['role', 'api_url', 'conn_url', 'state']:
if key not in new_data:
logger.warning('Could not register service: not enough params in member data')
return
if old_data.get(key) != new_data[key]:
update = True
if force or update:
return self._update_service(new_data)
@catch_consul_errors
def _do_attempt_to_acquire_leader(self, permanent):
try:
kwargs = {} if permanent else {'acquire': self._session}
return self.retry(self._client.kv.put, self.leader_path, self._name, **kwargs)
except InvalidSession:
self._session = None
logger.error('Our session disappeared from Consul. Will try to get a new one and retry attempt')
self.refresh_session()
return self.retry(self._client.kv.put, self.leader_path, self._name, acquire=self._session)
def attempt_to_acquire_leader(self, permanent=False):
if not self._session and not permanent:
self.refresh_session()
ret = self._do_attempt_to_acquire_leader(permanent)
args = {} if permanent else {'acquire': self._session}
ret = self.retry(self._client.kv.put, self.leader_path, self._name, **args)
if not ret:
logger.info('Could not take out TTL lock')
return ret
def take_leader(self):
@@ -465,7 +271,7 @@ class Consul(AbstractDCS):
return self._client.kv.put(self.leader_optime_path, last_operation)
@catch_consul_errors
def _update_leader(self):
def update_leader(self):
if self._session:
self.retry(self._client.session.renew, self._session)
self._last_session_refresh = time.time()
@@ -484,10 +290,6 @@ class Consul(AbstractDCS):
def delete_cluster(self):
return self.retry(self._client.kv.delete, self.client_path(''), recurse=True)
@catch_consul_errors
def set_history_value(self, value):
return self._client.kv.put(self.history_path, value)
@catch_consul_errors
def delete_leader(self):
cluster = self.cluster
@@ -496,11 +298,11 @@ class Consul(AbstractDCS):
@catch_consul_errors
def set_sync_state_value(self, value, index=None):
return self.retry(self._client.kv.put, self.sync_path, value, cas=index)
return self._client.kv.put(self.sync_path, value, cas=index)
@catch_consul_errors
def delete_sync_state(self, index=None):
return self.retry(self._client.kv.delete, self.sync_path, cas=index)
return self._client.kv.delete(self.sync_path, cas=index)
def watch(self, leader_index, timeout):
if self.__do_not_watch:
@@ -514,12 +316,11 @@ class Consul(AbstractDCS):
idx, _ = self._client.kv.get(self.leader_path, index=leader_index, wait=str(timeout) + 's')
return str(idx) != str(leader_index)
except (ConsulException, HTTPException, HTTPError, socket.error, socket.timeout):
logger.exception('watch')
logging.exception('watch')
timeout = end_time - time.time()
try:
return super(Consul, self).watch(None, timeout)
finally:
self._last_session_refresh = 0
self.event.clear()
+17 -50
View File
@@ -1,20 +1,18 @@
from __future__ import absolute_import
import etcd
import json
import logging
import os
import urllib3.util.connection
import random
import requests
import six
import socket
import time
from dns.exception import DNSException
from dns import resolver
from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
from patroni.exceptions import DCSError
from patroni.utils import Retry, RetryFailedError, split_host_port
from patroni.utils import Retry, RetryFailedError
from urllib3.exceptions import HTTPError, ReadTimeoutError
from requests.exceptions import RequestException
from six.moves.queue import Queue
@@ -25,10 +23,6 @@ from threading import Thread
logger = logging.getLogger(__name__)
def uri(protocol, host, port, endpoint=''):
return '{0}://{1}:{2}{3}'.format(protocol, host, port, endpoint)
class EtcdError(DCSError):
pass
@@ -210,11 +204,8 @@ class Client(etcd.Client):
self._machines_cache = self.machines
if self._base_uri in self._machines_cache:
self._machines_cache.remove(self._base_uri)
except etcd.EtcdConnectionFailed as e:
if isinstance(e, etcd.EtcdWatchTimedOut) and self._machines_cache:
self._base_uri = self._next_server()
else:
self._update_machines_cache = True
except etcd.EtcdConnectionFailed:
self._update_machines_cache = True
if not response:
raise
return self._handle_server_response(response)
@@ -236,7 +227,7 @@ class Client(etcd.Client):
protocol = 'https' if '-ssl' in r else 'http'
endpoint = '/members' if '-server' in r else ''
for host, port in self.get_srv_record('_etcd{0}._tcp.{1}'.format(r, srv)):
url = uri(protocol, host, port, endpoint)
url = '{0}://{1}:{2}{3}'.format(protocol, host, port, endpoint)
if endpoint:
try:
response = requests.get(url, timeout=self.read_timeout, verify=False)
@@ -263,10 +254,10 @@ class Client(etcd.Client):
host, port = sa[:2]
if af == socket.AF_INET6:
host = '[{0}]'.format(host)
ret.append(uri(self.protocol, host, port))
ret.append('{0}://{1}:{2}'.format(self.protocol, host, port))
if ret:
return list(set(ret))
return [uri(self.protocol, host, port)]
return ['{0}://{1}:{2}'.format(self.protocol, host, port)]
def _load_machines_cache(self):
"""This method should fill up `_machines_cache` from scratch.
@@ -276,20 +267,17 @@ class Client(etcd.Client):
self._update_machines_cache = True
if 'srv' not in self._config and 'host' not in self._config and 'hosts' not in self._config:
raise Exception('Neither srv, hosts, host nor url are defined in etcd section of config')
if 'srv' not in self._config and 'host' not in self._config:
raise Exception('Neither srv nor host url are defined in etcd section of config')
if self._use_proxies:
self._machines_cache = [uri(self.protocol, self._config['host'], self._config['port'])]
self._machines_cache = ['{0}://{1}:{2}'.format(self.protocol, self._config['host'], self._config['port'])]
else:
self._machines_cache = []
if 'srv' in self._config:
self._machines_cache = self._get_machines_cache_from_srv(self._config['srv'])
if not self._machines_cache and 'hosts' in self._config:
self._machines_cache = list(self._config['hosts'])
if not self._machines_cache and 'host' in self._config:
self._machines_cache = self._get_machines_cache_from_dns(self._config['host'], self._config['port'])
@@ -359,20 +347,8 @@ class Etcd(AbstractDCS):
r = urlparse(config['url'])
config.update({'protocol': r.scheme, 'host': r.hostname, 'port': r.port or 2379,
'username': r.username, 'password': r.password})
elif 'hosts' in config:
hosts = config.pop('hosts')
default_port = config.pop('port', 2379)
protocol = config.get('protocol', 'http')
if isinstance(hosts, six.string_types):
hosts = hosts.split(',')
config['hosts'] = []
for value in hosts:
if isinstance(value, six.string_types):
config['hosts'].append(uri(protocol, *split_host_port(value, default_port)))
elif 'host' in config:
host, port = split_host_port(config['host'], 2379)
host, port = (config['host'] + ':2379').split(':')[:2]
config['host'] = host
if 'port' not in config:
config['port'] = int(port)
@@ -448,7 +424,7 @@ class Etcd(AbstractDCS):
def _load_cluster(self):
try:
result = self.retry(self._client.read, self.client_path(''), recursive=True)
nodes = {os.path.relpath(node.key, result.key).replace('\\', '/'): node for node in result.leaves}
nodes = {os.path.relpath(node.key, result.key): node for node in result.leaves}
# get initialize flag
initialize = nodes.get(self._INITIALIZE)
@@ -458,10 +434,6 @@ class Etcd(AbstractDCS):
config = nodes.get(self._CONFIG)
config = config and ClusterConfig.from_node(config.modifiedIndex, config.value)
# get timeline history
history = nodes.get(self._HISTORY)
history = history and TimelineHistory.from_node(history.modifiedIndex, history.value)
# get last leader operation
last_leader_operation = nodes.get(self._LEADER_OPTIME)
last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation.value)
@@ -486,17 +458,16 @@ class Etcd(AbstractDCS):
sync = nodes.get(self._SYNC)
sync = SyncState.from_node(sync and sync.modifiedIndex, sync and sync.value)
self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
except etcd.EtcdKeyNotFound:
self._cluster = Cluster(None, None, None, None, [], None, None, None)
self._cluster = Cluster(None, None, None, None, [], None, None)
except Exception as e:
self._handle_exception(e, 'get_cluster', raise_ex=EtcdError('Etcd is not responding properly'))
self._has_failed = False
@catch_etcd_errors
def touch_member(self, data, ttl=None, permanent=False):
data = json.dumps(data, separators=(',', ':'))
return self._client.set(self.member_path, data, None if permanent else ttl or self._ttl)
return self.retry(self._client.set, self.member_path, data, None if permanent else ttl or self._ttl)
@catch_etcd_errors
def take_leader(self):
@@ -528,7 +499,7 @@ class Etcd(AbstractDCS):
return self._client.set(self.leader_optime_path, last_operation)
@catch_etcd_errors
def _update_leader(self):
def update_leader(self):
return self.retry(self._client.test_and_set, self.leader_path, self._name, self._name, self._ttl)
@catch_etcd_errors
@@ -547,13 +518,9 @@ class Etcd(AbstractDCS):
def delete_cluster(self):
return self.retry(self._client.delete, self.client_path(''), recursive=True)
@catch_etcd_errors
def set_history_value(self, value):
return self._client.write(self.history_path, value)
@catch_etcd_errors
def set_sync_state_value(self, value, index=None):
return self.retry(self._client.write, self.sync_path, value, prevIndex=index or 0)
return self._client.write(self.sync_path, value, prevIndex=index or 0)
@catch_etcd_errors
def delete_sync_state(self, index=None):
-434
View File
@@ -1,434 +0,0 @@
from __future__ import absolute_import
import datetime
import functools
import json
import logging
import socket
import sys
import time
from kubernetes import client as k8s_client, config as k8s_config, watch as k8s_watch
from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
from patroni.exceptions import DCSError
from patroni.utils import deep_compare, tzutc, Retry, RetryFailedError
from urllib3.exceptions import HTTPError
from six.moves.http_client import HTTPException
logger = logging.getLogger(__name__)
class KubernetesError(DCSError):
pass
class KubernetesRetriableException(k8s_client.rest.ApiException):
def __init__(self, orig):
super(KubernetesRetriableException, self).__init__(orig.status, orig.reason)
self.body = orig.body
self.headers = orig.headers
class CoreV1ApiProxy(object):
def __init__(self, use_endpoints=False):
self._api = k8s_client.CoreV1Api()
self._request_timeout = None
self._use_endpoints = use_endpoints
def set_timeout(self, timeout):
self._request_timeout = (1, timeout / 3.0)
def __getattr__(self, func):
if func.endswith('_kind'):
func = func[:-4] + ('endpoints' if self._use_endpoints else 'config_map')
def wrapper(*args, **kwargs):
if '_request_timeout' not in kwargs:
kwargs['_request_timeout'] = self._request_timeout
try:
return getattr(self._api, func)(*args, **kwargs)
except k8s_client.rest.ApiException as e:
if e.status in (502, 503, 504): # XXX
raise KubernetesRetriableException(e)
raise
return wrapper
def catch_kubernetes_errors(func):
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except k8s_client.rest.ApiException as e:
if e.status == 403:
logger.exception('Permission denied')
elif e.status != 409: # Object exists or conflict in resource_version
logger.exception('Unexpected error from Kubernetes API')
return False
except (RetryFailedError, HTTPException, HTTPError, socket.error, socket.timeout):
return False
return wrapper
class Kubernetes(AbstractDCS):
def __init__(self, config):
self._labels = config['labels']
self._labels[config.get('scope_label', 'cluster-name')] = config['scope']
self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
self._namespace = config.get('namespace') or 'default'
self._role_label = config.get('role_label', 'role')
config['namespace'] = ''
super(Kubernetes, self).__init__(config)
self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
retry_exceptions=(KubernetesRetriableException, HTTPException,
HTTPError, socket.error, socket.timeout))
self._ttl = None
try:
k8s_config.load_incluster_config()
except k8s_config.ConfigException:
k8s_config.load_kube_config(context=config.get('context', 'local'))
self.__subsets = None
use_endpoints = config.get('use_endpoints') and (config.get('patronictl') or 'pod_ip' in config)
if use_endpoints:
addresses = [k8s_client.V1EndpointAddress(ip='127.0.0.1' if config.get('patronictl') else config['pod_ip'])]
ports = []
for p in config.get('ports', [{}]):
port = {'port': int(p.get('port', '5432'))}
port.update({n: p[n] for n in ('name', 'protocol') if p.get(n)})
ports.append(k8s_client.V1EndpointPort(**port))
self.__subsets = [k8s_client.V1EndpointSubset(addresses=addresses, ports=ports)]
self._should_create_config_service = True
self._api = CoreV1ApiProxy(use_endpoints)
self.set_retry_timeout(config['retry_timeout'])
self.set_ttl(config.get('ttl') or 30)
self._leader_observed_record = {}
self._leader_observed_time = None
self._leader_resource_version = None
self._leader_observed_subsets = []
self.__do_not_watch = False
def retry(self, *args, **kwargs):
return self._retry.copy()(*args, **kwargs)
def client_path(self, path):
return super(Kubernetes, self).client_path(path)[1:].replace('/', '-')
@property
def leader_path(self):
return self._base_path[1:] if self.__subsets else super(Kubernetes, self).leader_path
def set_ttl(self, ttl):
ttl = int(ttl)
self.__do_not_watch = self._ttl != ttl
self._ttl = ttl
def set_retry_timeout(self, retry_timeout):
self._retry.deadline = retry_timeout
self._api.set_timeout(retry_timeout)
@staticmethod
def member(pod):
annotations = pod.metadata.annotations or {}
member = Member.from_node(pod.metadata.resource_version, pod.metadata.name, None, annotations.get('status', ''))
member.data['pod_labels'] = pod.metadata.labels
return member
def _load_cluster(self):
try:
# get list of members
response = self.retry(self._api.list_namespaced_pod, self._namespace, label_selector=self._label_selector)
members = [self.member(pod) for pod in response.items]
response = self.retry(self._api.list_namespaced_kind, self._namespace, label_selector=self._label_selector)
nodes = {item.metadata.name: item for item in response.items}
config = nodes.get(self.config_path)
metadata = config and config.metadata
annotations = metadata and metadata.annotations or {}
# get initialize flag
initialize = annotations.get(self._INITIALIZE)
# get global dynamic configuration
config = ClusterConfig.from_node(metadata and metadata.resource_version,
annotations.get(self._CONFIG) or '{}',
metadata.resource_version if self._CONFIG in annotations else 0)
# get timeline history
history = TimelineHistory.from_node(metadata and metadata.resource_version,
annotations.get(self._HISTORY) or '[]')
leader = nodes.get(self.leader_path)
metadata = leader and leader.metadata
self._leader_resource_version = metadata.resource_version if metadata else None
self._leader_observed_subsets = leader.subsets if self.__subsets and leader and leader.subsets else []
annotations = metadata and metadata.annotations or {}
# get last leader operation
last_leader_operation = annotations.get(self._OPTIME)
last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation)
# get leader
leader_record = {n: annotations.get(n) for n in (self._LEADER, 'acquireTime',
'ttl', 'renewTime', 'transitions') if n in annotations}
if (leader_record or self._leader_observed_record) and leader_record != self._leader_observed_record:
self._leader_observed_record = leader_record
self._leader_observed_time = time.time()
leader = leader_record.get(self._LEADER)
try:
ttl = int(leader_record.get('ttl')) or self._ttl
except (TypeError, ValueError):
ttl = self._ttl
if not metadata or not self._leader_observed_time or self._leader_observed_time + ttl < time.time():
leader = None
if metadata:
member = Member(-1, leader, None, {})
member = ([m for m in members if m.name == leader] or [member])[0]
leader = Leader(response.metadata.resource_version, None, member)
# failover key
failover = nodes.get(self.failover_path)
metadata = failover and failover.metadata
failover = Failover.from_node(metadata and metadata.resource_version, metadata and metadata.annotations)
# get synchronization state
sync = nodes.get(self.sync_path)
metadata = sync and sync.metadata
sync = SyncState.from_node(metadata and metadata.resource_version, metadata and metadata.annotations)
self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
except Exception:
logger.exception('get_cluster')
raise KubernetesError('Kubernetes API is not responding properly')
@staticmethod
def compare_ports(p1, p2):
return p1.name == p2.name and p1.port == p2.port and (p1.protocol or 'TCP') == (p2.protocol or 'TCP')
@staticmethod
def subsets_changed(last_observed_subsets, subsets):
"""
>>> Kubernetes.subsets_changed([], [])
False
>>> Kubernetes.subsets_changed([], [k8s_client.V1EndpointSubset()])
True
>>> s1 = [k8s_client.V1EndpointSubset(addresses=[k8s_client.V1EndpointAddress(ip='1.2.3.4')])]
>>> s2 = [k8s_client.V1EndpointSubset(addresses=[k8s_client.V1EndpointAddress(ip='1.2.3.5')])]
>>> Kubernetes.subsets_changed(s1, s2)
True
>>> a = [k8s_client.V1EndpointAddress(ip='1.2.3.4')]
>>> s1 = [k8s_client.V1EndpointSubset(addresses=a, ports=[k8s_client.V1EndpointPort(protocol='TCP', port=1)])]
>>> s2 = [k8s_client.V1EndpointSubset(addresses=a, ports=[k8s_client.V1EndpointPort(port=5432)])]
>>> Kubernetes.subsets_changed(s1, s2)
True
>>> p1 = k8s_client.V1EndpointPort(name='port1', port=1)
>>> p2 = k8s_client.V1EndpointPort(name='port2', port=2)
>>> p3 = k8s_client.V1EndpointPort(name='port3', port=3)
>>> s1 = [k8s_client.V1EndpointSubset(addresses=a, ports=[p1, p2])]
>>> s2 = [k8s_client.V1EndpointSubset(addresses=a, ports=[p2, p3])]
>>> Kubernetes.subsets_changed(s1, s2)
True
>>> s2 = [k8s_client.V1EndpointSubset(addresses=a, ports=[p2, p1])]
>>> Kubernetes.subsets_changed(s1, s2)
False
"""
if len(last_observed_subsets) != len(subsets):
return True
if subsets == []:
return False
if len(last_observed_subsets[0].addresses or []) != 1 or \
last_observed_subsets[0].addresses[0].ip != subsets[0].addresses[0].ip or \
len(last_observed_subsets[0].ports) != len(subsets[0].ports):
return True
if len(subsets[0].ports) == 1:
return not Kubernetes.compare_ports(last_observed_subsets[0].ports[0], subsets[0].ports[0])
observed_ports = {p.name: p for p in last_observed_subsets[0].ports}
for p in subsets[0].ports:
if p.name not in observed_ports or not Kubernetes.compare_ports(p, observed_ports.pop(p.name)):
return True
return False
@catch_kubernetes_errors
def patch_or_create(self, name, annotations, resource_version=None, patch=False, retry=True, subsets=None):
metadata = {'namespace': self._namespace, 'name': name, 'labels': self._labels, 'annotations': annotations}
if patch or resource_version:
if resource_version is not None:
metadata['resource_version'] = resource_version
func = functools.partial(self._api.patch_namespaced_kind, name)
else:
func = functools.partial(self._api.create_namespaced_kind)
# skip annotations with null values
metadata['annotations'] = {k: v for k, v in metadata['annotations'].items() if v is not None}
metadata = k8s_client.V1ObjectMeta(**metadata)
if subsets is not None and self.__subsets:
endpoints = {'metadata': metadata}
if self.subsets_changed(self._leader_observed_subsets, subsets):
endpoints['subsets'] = subsets
body = k8s_client.V1Endpoints(**endpoints)
else:
body = k8s_client.V1ConfigMap(metadata=metadata)
return self.retry(func, self._namespace, body) if retry else func(self._namespace, body)
def patch_or_create_config(self, annotations, resource_version=None, patch=False, retry=True):
# SCOPE-config endpoint requires corresponding service otherwise it might be "cleaned" by k8s master
if self.__subsets and not patch and not resource_version:
self._should_create_config_service = True
self._create_config_service()
return self.patch_or_create(self.config_path, annotations, resource_version, patch, retry)
def _create_config_service(self):
metadata = k8s_client.V1ObjectMeta(namespace=self._namespace, name=self.config_path, labels=self._labels)
body = k8s_client.V1Service(metadata=metadata, spec=k8s_client.V1ServiceSpec(cluster_ip='None'))
try:
if not self._api.create_namespaced_service(self._namespace, body):
return
except Exception as e:
if not isinstance(e, k8s_client.rest.ApiException) or e.status != 409: # Service already exists
return logger.exception('create_config_service failed')
self._should_create_config_service = False
def _write_leader_optime(self, last_operation):
"""Unused"""
def _update_leader(self):
"""Unused"""
def update_leader(self, last_operation, access_is_restricted=False):
now = datetime.datetime.now(tzutc).isoformat()
annotations = {self._LEADER: self._name, 'ttl': str(self._ttl), 'renewTime': now,
'acquireTime': self._leader_observed_record.get('acquireTime') or now,
'transitions': self._leader_observed_record.get('transitions') or '0'}
if last_operation:
annotations[self._OPTIME] = last_operation
subsets = [] if access_is_restricted else self.__subsets
ret = self.patch_or_create(self.leader_path, annotations, self._leader_resource_version, subsets=subsets)
if ret:
self._leader_resource_version = ret.metadata.resource_version
return ret
def attempt_to_acquire_leader(self, permanent=False):
now = datetime.datetime.now(tzutc).isoformat()
annotations = {self._LEADER: self._name, 'ttl': str(sys.maxsize if permanent else self._ttl),
'renewTime': now, 'acquireTime': now, 'transitions': '0'}
if self._leader_observed_record:
try:
transitions = int(self._leader_observed_record.get('transitions'))
except (TypeError, ValueError):
transitions = 0
if self._leader_observed_record.get(self._LEADER) != self._name:
transitions += 1
else:
annotations['acquireTime'] = self._leader_observed_record.get('acquireTime') or now
annotations['transitions'] = str(transitions)
subsets = [] if self.__subsets else None
ret = self.patch_or_create(self.leader_path, annotations, self._leader_resource_version, subsets=subsets)
if ret:
self._leader_resource_version = ret.metadata.resource_version
else:
logger.info('Could not take out TTL lock')
return ret
def take_leader(self):
return self.attempt_to_acquire_leader()
def set_failover_value(self, value, index=None):
"""Unused"""
def manual_failover(self, leader, candidate, scheduled_at=None, index=None):
annotations = {'leader': leader or None, 'member': candidate or None, 'scheduled_at': scheduled_at}
patch = bool(self.cluster and isinstance(self.cluster.failover, Failover) and self.cluster.failover.index)
return self.patch_or_create(self.failover_path, annotations, index, bool(index or patch), False)
def set_config_value(self, value, index=None):
patch = bool(index or self.cluster and self.cluster.config and self.cluster.config.index)
return self.patch_or_create_config({self._CONFIG: value}, index, patch, False)
@catch_kubernetes_errors
def touch_member(self, data, ttl=None, permanent=False):
cluster = self.cluster
if cluster and cluster.leader and cluster.leader.name == self._name:
role = 'promoted' if data['role'] in ('replica', 'promoted') else 'master'
elif data['state'] == 'running' and data['role'] != 'master':
role = data['role']
else:
role = None
member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
pod_labels = member and member.data.pop('pod_labels', None)
ret = pod_labels is not None and pod_labels.get(self._role_label) == role and deep_compare(data, member.data)
if not ret:
metadata = {'namespace': self._namespace, 'name': self._name, 'labels': {self._role_label: role},
'annotations': {'status': json.dumps(data, separators=(',', ':'))}}
body = k8s_client.V1Pod(metadata=k8s_client.V1ObjectMeta(**metadata))
ret = self._api.patch_namespaced_pod(self._name, self._namespace, body)
if self.__subsets and self._should_create_config_service:
self._create_config_service()
return ret
def initialize(self, create_new=True, sysid=""):
cluster = self.cluster
resource_version = cluster.config.index if cluster and cluster.config and cluster.config.index else None
return self.patch_or_create_config({self._INITIALIZE: sysid}, resource_version)
def delete_leader(self):
if self.cluster and isinstance(self.cluster.leader, Leader) and self.cluster.leader.name == self._name:
self.patch_or_create(self.leader_path, {self._LEADER: None}, self._leader_resource_version, True, False, [])
self.reset_cluster()
def cancel_initialization(self):
self.patch_or_create_config({self._INITIALIZE: None}, self.cluster.config.index, True)
@catch_kubernetes_errors
def delete_cluster(self):
self.retry(self._api.delete_collection_namespaced_kind, self._namespace, label_selector=self._label_selector)
def set_history_value(self, value):
patch = bool(self.cluster and self.cluster.config and self.cluster.config.index)
return self.patch_or_create_config({self._HISTORY: value}, None, patch, False)
def set_sync_state_value(self, value, index=None):
"""Unused"""
def write_sync_state(self, leader, sync_standby, index=None):
return self.patch_or_create(self.sync_path, self.sync_state(leader, sync_standby), index, False)
def delete_sync_state(self, index=None):
return self.write_sync_state(None, None, index)
def watch(self, leader_index, timeout):
if self.__do_not_watch:
self.__do_not_watch = False
return True
if leader_index:
end_time = time.time() + timeout
w = k8s_watch.Watch()
while timeout >= 1:
try:
for event in w.stream(self._api.list_namespaced_kind, self._namespace,
resource_version=leader_index, timeout_seconds=int(timeout + 0.5),
field_selector='metadata.name=' + self.leader_path,
_request_timeout=(1, timeout + 1)):
return event['raw_object'].get('metadata', {}).get('resourceVersion') != leader_index
return False
except KeyboardInterrupt:
raise
except Exception:
logger.exception('watch')
timeout = end_time - time.time()
try:
return super(Kubernetes, self).watch(None, timeout)
finally:
self.event.clear()
+57 -64
View File
@@ -1,14 +1,11 @@
import json
import logging
import select
import time
from kazoo.client import KazooClient, KazooState, KazooRetry
from kazoo.exceptions import NoNodeError, NodeExistsError
from kazoo.handlers.threading import SequentialThreadingHandler
from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
from patroni.exceptions import DCSError
from patroni.utils import deep_compare
logger = logging.getLogger(__name__)
@@ -38,21 +35,12 @@ class PatroniSequentialThreadingHandler(SequentialThreadingHandler):
`connect_timeout` (negotiated session timeout) as the second element."""
args = list(args)
if len(args) == 0: # kazoo 2.6.0 slightly changed the way how it calls create_connection method
kwargs['timeout'] = max(self._connect_timeout, kwargs.get('timeout', self._connect_timeout*10)/10.0)
elif len(args) == 1:
if len(args) == 1:
args.append(self._connect_timeout)
else:
args[1] = max(self._connect_timeout, args[1]/10.0)
return super(PatroniSequentialThreadingHandler, self).create_connection(*args, **kwargs)
def select(self, *args, **kwargs):
"""Python3 raises `ValueError` if socket is closed, because fd == -1"""
try:
return super(PatroniSequentialThreadingHandler, self).select(*args, **kwargs)
except ValueError as e:
raise select.error(9, str(e))
class ZooKeeper(AbstractDCS):
@@ -69,6 +57,7 @@ class ZooKeeper(AbstractDCS):
max_delay=1, max_tries=-1, sleep_func=time.sleep))
self._client.add_listener(self.session_listener)
self._my_member_data = None
self._fetch_cluster = True
self._orig_kazoo_connect = self._client._connection._connect
@@ -170,10 +159,6 @@ class ZooKeeper(AbstractDCS):
config = self.get_node(self.config_path, watch=self.cluster_watcher) if self._CONFIG in nodes else None
config = config and ClusterConfig.from_node(config[1].version, config[0], config[1].mzxid)
# get timeline history
history = self.get_node(self.history_path, watch=self.cluster_watcher) if self._HISTORY in nodes else None
history = history and TimelineHistory.from_node(history[1].mzxid, history[0])
# get last leader operation
last_leader_operation = self._OPTIME in nodes and self._fetch_cluster and self.get_node(self.leader_optime_path)
last_leader_operation = last_leader_operation and int(last_leader_operation[0]) or 0
@@ -206,7 +191,7 @@ class ZooKeeper(AbstractDCS):
failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
failover = failover and Failover.from_node(failover[1].version, failover[0])
self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
def _load_cluster(self):
if self._fetch_cluster or self._cluster is None:
@@ -217,82 +202,73 @@ class ZooKeeper(AbstractDCS):
self.cluster_watcher(None)
raise ZooKeeperError('ZooKeeper in not responding properly')
def _create(self, path, value, retry=False, ephemeral=False):
def _create(self, path, value, **kwargs):
try:
if retry:
self._client.retry(self._client.create, path, value, makepath=True, ephemeral=ephemeral)
else:
self._client.create_async(path, value, makepath=True, ephemeral=ephemeral).get(timeout=1)
self._client.retry(self._client.create, path, value.encode('utf-8'), **kwargs)
return True
except Exception:
logger.exception('Failed to create %s', path)
return False
except:
return False
def attempt_to_acquire_leader(self, permanent=False):
ret = self._create(self.leader_path, self._name.encode('utf-8'), retry=True, ephemeral=not permanent)
ret = self._create(self.leader_path, self._name, makepath=True, ephemeral=not permanent)
if not ret:
logger.info('Could not take out TTL lock')
return ret
def _set_or_create(self, key, value, index=None, retry=False, do_not_create_empty=False):
value = value.encode('utf-8')
def set_failover_value(self, value, index=None):
try:
if retry:
self._client.retry(self._client.set, key, value, version=index or -1)
else:
self._client.set_async(key, value, version=index or -1).get(timeout=1)
self._client.retry(self._client.set, self.failover_path, value.encode('utf-8'), version=index or -1)
return True
except NoNodeError:
if do_not_create_empty and not value:
return True
elif index is None:
return self._create(key, value, retry)
else:
return False
except Exception:
logger.exception('Failed to update %s', key)
return False
def set_failover_value(self, value, index=None):
return self._set_or_create(self.failover_path, value, index)
return value == '' or (index is None and self._create(self.failover_path, value))
except:
logging.exception('set_failover_value')
return False
def set_config_value(self, value, index=None):
return self._set_or_create(self.config_path, value, index, retry=True)
try:
self._client.retry(self._client.set, self.config_path, value.encode('utf-8'), version=index or -1)
return True
except NoNodeError:
return index is None and self._create(self.config_path, value)
except Exception:
logging.exception('set_config_value')
return False
def initialize(self, create_new=True, sysid=""):
sysid = sysid.encode('utf-8')
return self._create(self.initialize_path, sysid, retry=True) if create_new \
else self._client.retry(self._client.set, self.initialize_path, sysid)
return self._create(self.initialize_path, sysid, makepath=True) if create_new \
else self._client.retry(self._client.set, self.initialize_path, sysid.encode("utf-8"))
def touch_member(self, data, ttl=None, permanent=False):
cluster = self.cluster
member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
encoded_data = json.dumps(data, separators=(',', ':')).encode('utf-8')
data = data.encode('utf-8')
if member and self._client.client_id is not None and member.session != self._client.client_id[0]:
try:
self._client.delete_async(self.member_path).get(timeout=1)
except NoNodeError:
pass
except Exception:
except:
return False
member = None
if member:
if deep_compare(data, member.data):
if data == self._my_member_data:
return True
else:
try:
self._client.create_async(self.member_path, encoded_data, makepath=True,
ephemeral=not permanent).get(timeout=1)
self._client.create_async(self.member_path, data, makepath=True, ephemeral=not permanent).get(timeout=1)
self._my_member_data = data
return True
except Exception as e:
if not isinstance(e, NodeExistsError):
logger.exception('touch_member')
return False
try:
self._client.set_async(self.member_path, encoded_data).get(timeout=1)
self._client.set_async(self.member_path, data).get(timeout=1)
self._my_member_data = data
return True
except Exception:
except:
logger.exception('touch_member')
return False
@@ -301,13 +277,26 @@ class ZooKeeper(AbstractDCS):
return self.attempt_to_acquire_leader()
def _write_leader_optime(self, last_operation):
return self._set_or_create(self.leader_optime_path, last_operation)
last_operation = last_operation.encode('utf-8')
try:
self._client.set_async(self.leader_optime_path, last_operation).get(timeout=1)
return True
except NoNodeError:
try:
self._client.create_async(self.leader_optime_path, last_operation, makepath=True).get(timeout=1)
return True
except:
logger.exception('Failed to create %s', self.leader_optime_path)
except:
logger.exception('Failed to update %s', self.leader_optime_path)
return False
def _update_leader(self):
def update_leader(self):
return True
def delete_leader(self):
self._client.restart()
self._my_member_data = None
return True
def _cancel_initialization(self):
@@ -318,7 +307,7 @@ class ZooKeeper(AbstractDCS):
def cancel_initialization(self):
try:
self._client.retry(self._cancel_initialization)
except Exception:
except:
logger.exception("Unable to delete initialize key")
def delete_cluster(self):
@@ -327,11 +316,15 @@ class ZooKeeper(AbstractDCS):
except NoNodeError:
return True
def set_history_value(self, value):
return self._set_or_create(self.history_path, value)
def set_sync_state_value(self, value, index=None):
return self._set_or_create(self.sync_path, value, index, retry=True, do_not_create_empty=True)
try:
self._client.retry(self._client.set, self.sync_path, value.encode('utf-8'), version=index or -1)
return True
except NoNodeError:
return value == '' or (index is None and self._create(self.sync_path, value))
except:
logging.exception('set_sync_state_value')
return False
def delete_sync_state(self, index=None):
return self.set_sync_state_value("{}", index)
+79 -346
View File
@@ -6,42 +6,36 @@ import psycopg2
import requests
import sys
import time
import uuid
from collections import namedtuple
from multiprocessing.pool import ThreadPool
from patroni.async_executor import AsyncExecutor, CriticalTask
from patroni.exceptions import DCSError, PostgresConnectionException, PatroniException
from patroni.postgresql import ACTION_ON_START, ACTION_ON_ROLE_CHANGE
from patroni.postgresql import ACTION_ON_START
from patroni.utils import polling_loop, tzutc
from patroni.dcs import RemoteMember
from threading import RLock
logger = logging.getLogger(__name__)
class _MemberStatus(namedtuple('_MemberStatus', ['member', 'reachable', 'in_recovery', 'timeline',
'wal_position', 'tags', 'watchdog_failed'])):
class _MemberStatus(namedtuple('_MemberStatus', 'member,reachable,in_recovery,wal_position,tags,watchdog_failed')):
"""Node status distilled from API response:
member - dcs.Member object of the node
reachable - `!False` if the node is not reachable or is not responding with correct JSON
in_recovery - `!True` if pg_is_in_recovery() == true
timeline - timeline value from JSON
wal_position - maximum value of `replayed_location` or `received_location` from JSON
wal_position - value of `replayed_location` or `location` from JSON, dependin on its role.
tags - dictionary with values of different tags (i.e. nofailover)
watchdog_failed - indicates that watchdog is required by configuration but not available or failed
"""
@classmethod
def from_api_response(cls, member, json):
is_master = json['role'] == 'master'
timeline = json.get('timeline', 0)
wal = not is_master and max(json['xlog'].get('received_location', 0), json['xlog'].get('replayed_location', 0))
return cls(member, True, not is_master, timeline, wal, json.get('tags', {}), json.get('watchdog_failed', False))
return cls(member, True, not is_master, wal, json.get('tags', {}), json.get('watchdog_failed', False))
@classmethod
def unknown(cls, member):
return cls(member, False, None, 0, 0, {}, False)
return cls(member, False, None, 0, {}, False)
def failover_limitation(self):
"""Returns reason why this node can't promote or None if everything is ok."""
@@ -62,16 +56,10 @@ class Ha(object):
self.dcs = patroni.dcs
self.cluster = None
self.old_cluster = None
self._is_leader = False
self._is_leader_lock = RLock()
self._leader_access_is_restricted = False
self._was_paused = False
self._leader_timeline = None
self.recovering = False
self._post_bootstrap_task = None
self._crash_recovery_executed = False
self._start_timeout = None
self._async_executor = AsyncExecutor(self.state_handler, self.wakeup)
self._async_executor = AsyncExecutor(self.wakeup)
self.watchdog = patroni.watchdog
# Each member publishes various pieces of information to the DCS using touch_member. This lock protects
@@ -81,46 +69,8 @@ class Ha(object):
# standby. Changes protected by _member_state_lock.
self._disable_sync = 0
# We need following property to avoid shutdown of postgres when join of Patroni to the postgres
# already running as replica was aborted due to cluster not beeing initialized in DCS.
self._join_aborted = False
def check_mode(self, mode):
# Try to protect from the case when DCS was wiped out during pause
if self.cluster and self.cluster.config and self.cluster.config.modify_index:
return self.cluster.check_mode(mode)
else:
return self.patroni.config.check_mode(mode)
def is_paused(self):
return self.check_mode('pause')
def check_timeline(self):
return self.check_mode('check_timeline')
def get_standby_cluster_config(self):
if self.cluster and self.cluster.config and self.cluster.config.modify_index:
config = self.cluster.config.data
else:
config = self.patroni.config.dynamic_configuration
return config.get('standby_cluster')
def is_standby_cluster(self):
config = self.get_standby_cluster_config()
# Check whether or not provided configuration describes a standby cluster
return isinstance(config, dict) and (config.get('host') or config.get('port') or config.get('restore_command'))
def is_leader(self):
with self._is_leader_lock:
return self._is_leader and not self._leader_access_is_restricted
def set_is_leader(self, value):
with self._is_leader_lock:
self._is_leader = value
def set_leader_access_is_restricted(self, value):
with self._is_leader_lock:
self._leader_access_is_restricted = value
return self.cluster and self.cluster.is_paused()
def load_cluster_from_dcs(self):
cluster = self.dcs.get_cluster()
@@ -130,28 +80,18 @@ class Ha(object):
self.old_cluster = cluster
self.cluster = cluster
if self.cluster.is_unlocked() or self.cluster.leader.name != self.state_handler.name:
self.set_is_leader(False)
self._leader_timeline = None if cluster.is_unlocked() else cluster.leader.timeline
def acquire_lock(self):
self.set_leader_access_is_restricted(self.cluster.has_permanent_logical_slots(self.state_handler.name))
ret = self.dcs.attempt_to_acquire_leader()
self.set_is_leader(ret)
return ret
return self.dcs.attempt_to_acquire_leader()
def update_lock(self, write_leader_optime=False):
last_operation = None
if write_leader_optime:
try:
last_operation = self.state_handler.last_operation()
except Exception:
logger.exception('Exception when called state_handler.last_operation()')
ret = self.dcs.update_leader(last_operation, self._leader_access_is_restricted)
self.set_is_leader(ret)
ret = self.dcs.update_leader()
if ret:
self.watchdog.keepalive()
if write_leader_optime:
try:
self.dcs.write_leader_optime(self.state_handler.last_operation())
except:
pass
return ret
def has_lock(self):
@@ -175,35 +115,22 @@ class Ha(object):
'state': self.state_handler.state,
'role': self.state_handler.role
}
# following two lines are mainly necessary for consul, to avoid creation of master service
if data['role'] == 'master' and not self.is_leader():
data['role'] = 'promoted'
tags = self.get_effective_tags()
if tags:
data['tags'] = tags
if self.state_handler.pending_restart:
data['pending_restart'] = True
if self._async_executor.scheduled_action in (None, 'promote') \
and data['state'] in ['running', 'restarting', 'starting']:
if not self._async_executor.busy and data['state'] in ['running', 'restarting', 'starting']:
try:
timeline, wal_position = self.state_handler.timeline_wal_position()
data['xlog_location'] = wal_position
if not timeline:
timeline = self.state_handler.replica_cached_timeline(self._leader_timeline)
if timeline:
data['timeline'] = timeline
except Exception:
data['xlog_location'] = self.state_handler.wal_position(retry=False)
except:
pass
if self.patroni.scheduled_restart:
scheduled_restart_data = self.patroni.scheduled_restart.copy()
scheduled_restart_data['schedule'] = scheduled_restart_data['schedule'].isoformat()
data['scheduled_restart'] = scheduled_restart_data
if self.is_paused():
data['pause'] = True
return self.dcs.touch_member(data)
return self.dcs.touch_member(json.dumps(data, separators=(',', ':')))
def clone(self, clone_member=None, msg='(without leader)'):
if self.state_handler.clone(clone_member):
@@ -223,24 +150,14 @@ class Ha(object):
self._async_executor.schedule('bootstrap {0}'.format(msg))
self._async_executor.run_async(self.clone, args=(clone_member, msg))
return 'trying to bootstrap {0}'.format(msg)
# no initialize key and node is allowed to be master and has 'bootstrap' section in a configuration file
elif self.cluster.initialize is None and not self.patroni.nofailover and 'bootstrap' in self.patroni.config:
if self.dcs.initialize(create_new=True): # race for initialization
self.state_handler.bootstrapping = True
self._post_bootstrap_task = CriticalTask()
if self.is_standby_cluster():
self._async_executor.schedule('bootstrap_standby_leader')
self._async_executor.run_async(self.bootstrap_standby_leader)
return 'trying to bootstrap a new standby leader'
else:
self._async_executor.schedule('bootstrap')
self._async_executor.run_async(
self.state_handler.bootstrap,
args=(self.patroni.config['bootstrap'],)
)
return 'trying to bootstrap a new cluster'
self._async_executor.schedule('bootstrap')
self._async_executor.run_async(self.state_handler.bootstrap, args=(self.patroni.config['bootstrap'],))
return 'trying to bootstrap a new cluster'
else:
return 'failed to acquire initialize lock'
else:
@@ -251,34 +168,10 @@ class Ha(object):
return 'trying to ' + msg
return 'waiting for leader to bootstrap'
def bootstrap_standby_leader(self):
""" If we found 'standby' key in the configuration, we need to bootstrap
not a real master, but a 'standby leader', that will take base backup
from a remote master and start follow it.
"""
clone_source = self.get_remote_master()
msg = 'clone from remote master {0}'.format(clone_source.conn_url)
result = self.clone(clone_source, msg)
self._post_bootstrap_task.complete(result)
if result:
self.state_handler.set_role('standby_leader')
return result
def _handle_rewind_or_reinitialize(self):
leader = self.get_remote_master() if self.is_standby_cluster() else self.cluster.leader
if not self.state_handler.rewind_or_reinitialize_needed_and_possible(leader):
return None
if self.state_handler.can_rewind:
self._async_executor.schedule('running pg_rewind from ' + leader.name)
self._async_executor.run_async(self.state_handler.rewind, (leader,))
return True
# remove_data_directory_on_diverged_timelines is set
if not self.is_standby_cluster():
self._async_executor.schedule('reinitializing due to diverged timelines')
self._async_executor.run_async(self._do_reinitialize, args=(self.cluster, ))
def _handle_rewind(self):
if self.state_handler.rewind_needed_and_possible(self.cluster.leader):
self._async_executor.schedule('running pg_rewind from ' + self.cluster.leader.name)
self._async_executor.run_async(self.state_handler.rewind, (self.cluster.leader,))
return True
def recover(self):
@@ -290,44 +183,25 @@ class Ha(object):
if timeout == 0:
# We are requested to prefer failing over to restarting master. But see first if there
# is anyone to fail over to.
members = self.cluster.members
if self.is_synchronous_mode():
members = [m for m in members if self.cluster.sync.matches(m.name)]
if self.is_failover_possible(members):
if self.is_failover_possible(self.cluster.members):
logger.info("Master crashed. Failing over.")
self.demote('immediate')
return 'stopped PostgreSQL to fail over after a crash'
else:
timeout = None
data = self.state_handler.controldata()
if data.get('Database cluster state') in ('in production', 'shutting down', 'in crash recovery') and \
not self._crash_recovery_executed and (self.cluster.is_unlocked() or self.state_handler.can_rewind):
self._crash_recovery_executed = True
self._async_executor.schedule('doing crash recovery in a single user mode')
self._async_executor.run_async(self.state_handler.fix_cluster_state)
return self._async_executor.scheduled_action
self.load_cluster_from_dcs()
if self.is_standby_cluster() or not self.has_lock():
if not self.state_handler.rewind_executed:
self.state_handler.trigger_check_diverged_lsn()
if self._handle_rewind_or_reinitialize():
return self._async_executor.scheduled_action
if self.has_lock(): # in standby cluster
msg = "starting as a standby leader because i had the session lock"
node_to_follow = self._get_node_to_follow(self.cluster)
elif self.is_standby_cluster() and self.cluster.is_unlocked():
msg = "trying to follow a remote master because standby cluster is unhealthy"
node_to_follow = self.get_remote_master()
else:
msg = "starting as a secondary"
node_to_follow = self._get_node_to_follow(self.cluster)
elif self.has_lock():
if self.has_lock():
msg = "starting as readonly because i had the session lock"
node_to_follow = None
else:
if not self.state_handler.rewind_executed:
self.state_handler.trigger_check_diverged_lsn()
if self._handle_rewind():
return self._async_executor.scheduled_action
msg = "starting as a secondary"
node_to_follow = self._get_node_to_follow(self.cluster)
self.recovering = True
@@ -338,14 +212,10 @@ class Ha(object):
def _get_node_to_follow(self, cluster):
# determine the node to follow. If replicatefrom tag is set,
# try to follow the node mentioned there, otherwise, follow the leader.
is_leader = self.cluster.leader and self.state_handler.name == self.cluster.leader.name
if self.is_standby_cluster() and (is_leader or self.cluster.is_unlocked()):
node_to_follow = self.get_remote_master()
elif self.patroni.replicatefrom and self.patroni.replicatefrom != self.state_handler.name:
node_to_follow = cluster.get_member(self.patroni.replicatefrom)
else:
if not self.patroni.replicatefrom or self.patroni.replicatefrom == self.state_handler.name:
node_to_follow = cluster.leader
else:
node_to_follow = cluster.get_member(self.patroni.replicatefrom)
return node_to_follow if node_to_follow and node_to_follow.name != self.state_handler.name else None
@@ -358,8 +228,7 @@ class Ha(object):
node_to_follow = self._get_node_to_follow(self.cluster)
if self.is_paused():
if not (self.state_handler.need_rewind and self.state_handler.can_rewind_or_reinitialize_allowed)\
or self.cluster.is_unlocked():
if not (self.state_handler.need_rewind and self.state_handler.can_rewind) or self.cluster.is_unlocked():
self.state_handler.set_role('master' if is_leader else 'replica')
if is_leader:
return 'continue to run as master without lock'
@@ -369,7 +238,7 @@ class Ha(object):
self.demote('immediate-nolock')
return demote_reason
if self._handle_rewind_or_reinitialize():
if self._handle_rewind():
return self._async_executor.scheduled_action
if not self.state_handler.check_recovery_conf(node_to_follow):
@@ -379,10 +248,10 @@ class Ha(object):
return follow_reason
def is_synchronous_mode(self):
return self.check_mode('synchronous_mode')
return bool(self.cluster and self.cluster.config and self.cluster.config.data.get('synchronous_mode'))
def is_synchronous_mode_strict(self):
return self.check_mode('synchronous_mode_strict')
return bool(self.cluster and self.cluster.config and self.cluster.config.data.get('synchronous_mode_strict'))
def process_sync_replication(self):
"""Process synchronous standby beahvior.
@@ -470,47 +339,20 @@ class Ha(object):
with self._member_state_lock:
self._disable_sync -= 1
def update_cluster_history(self):
master_timeline = self.state_handler.get_master_timeline()
cluster_history = self.cluster.history and self.cluster.history.lines
if master_timeline == 1:
if cluster_history:
self.dcs.set_history_value('[]')
elif not cluster_history or cluster_history[-1][0] != master_timeline - 1 or len(cluster_history[-1]) != 4:
cluster_history = {l[0]: l for l in cluster_history or []}
history = self.state_handler.get_history(master_timeline)
if history:
for line in history:
# enrich current history with promotion timestamps stored in DCS
if len(line) == 3 and line[0] in cluster_history \
and len(cluster_history[line[0]]) == 4 \
and cluster_history[line[0]][1] == line[1]:
line.append(cluster_history[line[0]][3])
self.dcs.set_history_value(json.dumps(history, separators=(',', ':')))
def enforce_follow_remote_master(self, message):
self.state_handler.set_role('standby_leader')
demote_reason = 'cannot be a real master in standby cluster'
return self.follow(demote_reason, message)
def enforce_master_role(self, message, promote_message):
if not self.is_paused() and not self.watchdog.is_running and not self.watchdog.activate():
if self.state_handler.is_leader():
self.demote('immediate')
return 'Demoting self because watchdog could not be activated'
else:
self.release_leader_key_voluntarily()
return 'Not promoting self because watchdog could not be activated'
if not self.watchdog.is_running:
if not self.watchdog.activate():
if self.state_handler.is_leader():
self.demote('immediate')
return 'Demoting self because watchdog could not be activated'
else:
self.release_leader_key_voluntarily()
return 'Not promoting self because watchdog could not be actived'
if self.state_handler.is_leader():
if self.state_handler.is_leader() or self.state_handler.role == 'master':
# Inform the state handler about its master role.
# It may be unaware of it if postgres is promoted manually.
self.state_handler.set_role('master')
self.process_sync_replication()
self.update_cluster_history()
return message
elif self.state_handler.role == 'master':
self.process_sync_replication()
return message
else:
@@ -521,12 +363,8 @@ class Ha(object):
# Somebody else updated sync state, it may be due to us losing the lock. To be safe, postpone
# promotion until next cycle. TODO: trigger immediate retry of run_cycle
return 'Postponing promotion because synchronous replication state was updated by somebody else'
self.state_handler.set_synchronous_standby('*' if self.is_synchronous_mode_strict() else None)
if self.state_handler.role != 'master':
self.set_leader_access_is_restricted(self.cluster.has_permanent_logical_slots(self.state_handler.name))
self._async_executor.schedule('promote')
self._async_executor.run_async(self.state_handler.promote,
args=(self.dcs.loop_wait, self._leader_access_is_restricted))
self.state_handler.set_synchronous_standby(None)
self.state_handler.promote()
return promote_message
@staticmethod
@@ -540,7 +378,7 @@ class Ha(object):
logger.info('Got response from %s %s: %s', member.name, member.api_url, response.content)
return _MemberStatus.from_api_response(member, response.json())
except Exception as e:
logger.warning("Request failed to %s: GET %s (%s)", member.name, member.api_url, e)
logger.warning("request failed: GET %s (%s)", member.api_url, e)
return _MemberStatus.unknown(member)
def fetch_nodes_statuses(self, members):
@@ -557,23 +395,15 @@ class Ha(object):
:returns True when node is lagging
"""
lag = (self.cluster.last_leader_operation or 0) - wal_position
return lag > self.patroni.config.get('maximum_lag_on_failover', 0)
return lag > self.state_handler.config.get('maximum_lag_on_failover', 0)
def _is_healthiest_node(self, members, check_replication_lag=True):
"""This method tries to determine whether I am healthy enough to became a new leader candidate or not."""
_, my_wal_position = self.state_handler.timeline_wal_position()
my_wal_position = self.state_handler.wal_position()
if check_replication_lag and self.is_lagging(my_wal_position):
logger.info('My wal position exceeds maximum replication lag')
return False # Too far behind last reported wal position on master
if not self.is_standby_cluster() and self.check_timeline():
cluster_timeline = self.cluster.timeline
my_timeline = self.state_handler.replica_cached_timeline(cluster_timeline)
if my_timeline < cluster_timeline:
logger.info('My timeline %s is behind last known cluster timeline %s', my_timeline, cluster_timeline)
return False
# Prepare list of nodes to run check against
members = [m for m in members if m.name != self.state_handler.name and not m.nofailover and m.api_url]
@@ -584,13 +414,11 @@ class Ha(object):
logger.warning('Master (%s) is still alive', st.member.name)
return False
if my_wal_position < st.wal_position:
logger.info('Wal position of %s is ahead of my wal position', st.member.name)
return False
return True
def is_failover_possible(self, members):
ret = False
cluster_timeline = self.cluster.timeline
members = [m for m in members if m.name != self.state_handler.name and not m.nofailover and m.api_url]
if members:
for st in self.fetch_nodes_statuses(members):
@@ -599,9 +427,6 @@ class Ha(object):
logger.info('Member %s is %s', st.member.name, not_allowed_reason)
elif self.is_lagging(st.wal_position):
logger.info('Member %s exceeds maximum replication lag', st.member.name)
elif self.check_timeline() and (not st.timeline or st.timeline < cluster_timeline):
logger.info('Timeline %s of member %s is behind the cluster timeline %s',
st.timeline, st.member.name, cluster_timeline)
else:
ret = True
else:
@@ -711,21 +536,20 @@ class Ha(object):
PostgreSQL as quickly as possible without regard for data durability. May only be called synchronously.
"""
mode_control = {
'offline': dict(stop='fast', checkpoint=False, release=False, offline=True, async_req=False),
'graceful': dict(stop='fast', checkpoint=True, release=True, offline=False, async_req=False),
'immediate': dict(stop='immediate', checkpoint=False, release=True, offline=False, async_req=True),
'immediate-nolock': dict(stop='immediate', checkpoint=False, release=False, offline=False, async_req=True),
'offline': dict(stop='fast', checkpoint=False, release=False, offline=True, async=False),
'graceful': dict(stop='fast', checkpoint=True, release=True, offline=False, async=False),
'immediate': dict(stop='immediate', checkpoint=False, release=True, offline=False, async=True),
'immediate-nolock': dict(stop='immediate', checkpoint=False, release=False, offline=False, async=True),
}[mode]
self.state_handler.trigger_check_diverged_lsn()
self.state_handler.stop(mode_control['stop'], checkpoint=mode_control['checkpoint'],
on_safepoint=self.watchdog.disable if self.watchdog.is_running else None)
self.state_handler.set_role('demoted')
self.set_is_leader(False)
if mode_control['release']:
self.release_leader_key_voluntarily()
time.sleep(2) # Give a time to somebody to take the leader lock
self.release_leader_key_voluntarily()
time.sleep(2) # Give a time to somebody to take the leader lock
if mode_control['offline']:
node_to_follow, leader = None, None
else:
@@ -735,13 +559,11 @@ class Ha(object):
# FIXME: with mode offline called from DCS exception handler and handle_long_action_in_progress
# there could be an async action already running, calling follow from here will lead
# to racy state handler state updates.
if mode_control['async_req']:
if mode_control['async']:
self._async_executor.schedule('starting after demotion')
self._async_executor.run_async(self.state_handler.follow, (node_to_follow,))
else:
if self.is_synchronous_mode():
self.state_handler.set_synchronous_standby(None)
if self.state_handler.rewind_or_reinitialize_needed_and_possible(leader):
if self.state_handler.rewind_needed_and_possible(leader):
return False # do not start postgres, but run pg_rewind on the next iteration
self.state_handler.follow(node_to_follow)
@@ -799,16 +621,8 @@ class Ha(object):
if not failover.candidate and self.is_paused():
logger.warning('Failover is possible only to a specific candidate in a paused state')
else:
if self.is_synchronous_mode():
if failover.candidate and not self.cluster.sync.matches(failover.candidate):
logger.warning('Failover candidate=%s does not match with sync_standby=%s',
failover.candidate, self.cluster.sync.sync_standby)
members = []
else:
members = [m for m in self.cluster.members if self.cluster.sync.matches(m.name)]
else:
members = [m for m in self.cluster.members
if not failover.candidate or m.name == failover.candidate]
members = [m for m in self.cluster.members
if not failover.candidate or m.name == failover.candidate]
if self.is_failover_possible(members): # check that there are healthy members
self._async_executor.schedule('manual failover: demote')
self._async_executor.run_async(self.demote, ('graceful',))
@@ -838,18 +652,8 @@ class Ha(object):
logger.info('Cleaning up failover key after acquiring leader lock...')
self.dcs.manual_failover('', '')
self.load_cluster_from_dcs()
if self.is_standby_cluster():
# standby leader disappeared, and this is a healthiest
# replica, so it should become a new standby leader.
# This imply that we need to start following a remote master
msg = 'promoted self to a standby leader because i had the session lock'
return self.enforce_follow_remote_master(msg)
else:
return self.enforce_master_role(
'acquired session lock as a leader',
'promoted self to leader by acquiring session lock'
)
return self.enforce_master_role('acquired session lock as a leader',
'promoted self to leader by acquiring session lock')
else:
return self.follow('demoted self after trying and failing to obtain lock',
'following new leader after trying and failing to obtain lock')
@@ -876,33 +680,17 @@ class Ha(object):
self.dcs.reset_cluster()
return 'removed leader lock because postgres is not running as master'
if self.state_handler.is_leader() and self._leader_access_is_restricted:
self.state_handler.sync_replication_slots(self.cluster)
self.state_handler.call_nowait(ACTION_ON_ROLE_CHANGE)
self.set_leader_access_is_restricted(False)
if self.update_lock(True):
msg = self.process_manual_failover_from_leader()
if msg is not None:
return msg
if self.is_standby_cluster():
# in case of standby cluster we don't really need to
# enforce anything, since the leader is not a master.
# So just remind the role.
msg = 'no action. i am the standby leader with the lock'
return self.enforce_follow_remote_master(msg)
else:
return self.enforce_master_role(
'no action. i am the leader with the lock',
'promoted self to leader because i had the session lock'
)
return self.enforce_master_role('no action. i am the leader with the lock',
'promoted self to leader because i had the session lock')
else:
# Either there is no connection to DCS or someone else acquired the lock
logger.error('failed to update leader lock')
if self.state_handler.is_leader():
if self.is_paused():
return 'continue to run as master after failing to update leader lock in DCS'
self.demote('immediate-nolock')
return 'demoted self because failed to update leader lock in DCS'
else:
@@ -1026,14 +814,13 @@ class Ha(object):
def _do_reinitialize(self, cluster):
self.state_handler.stop('immediate')
# Commented redundant data directory cleanup here
# self.state_handler.remove_data_directory()
self.state_handler.remove_data_directory()
clone_member = self.cluster.get_clone_member(self.state_handler.name)
member_role = 'leader' if clone_member == self.cluster.leader else 'replica'
return self.clone(clone_member, "from {0} '{1}'".format(member_role, clone_member.name))
def reinitialize(self, force=False):
def reinitialize(self):
with self._async_executor:
self.load_cluster_from_dcs()
@@ -1043,11 +830,7 @@ class Ha(object):
if self.cluster.leader.name == self.state_handler.name:
return 'I am the leader, can not reinitialize'
if force:
self._async_executor.cancel()
with self._async_executor:
action = self._async_executor.schedule('reinitialize')
action = self._async_executor.schedule('reinitialize', immediately=True)
if action is not None:
return '{0} already in progress'.format(action)
@@ -1065,7 +848,7 @@ class Ha(object):
# background thread and has not even written a pid file yet.
with self._async_executor.critical_task as task:
if not task.cancel():
self.state_handler.terminate_starting_postmaster(postmaster=task.result)
self.state_handler.terminate_starting_postmaster(pid=task.result)
self.demote('immediate-nolock')
return 'lost leader lock during ' + self._async_executor.scheduled_action
@@ -1090,7 +873,6 @@ class Ha(object):
self.dcs.reset_cluster()
return 'removed leader key after trying and failing to start postgres'
return 'failed to start postgres'
self._crash_recovery_executed = False
return None
def cancel_initialization(self):
@@ -1120,12 +902,9 @@ class Ha(object):
if not self.watchdog.activate():
logger.error('Cancelling bootstrap because watchdog activation failed')
self.cancel_initialization()
self.state_handler.sync_replication_slots(self.cluster)
self.dcs.take_leader()
self.set_is_leader(True)
self.state_handler.call_nowait(ACTION_ON_START)
self.load_cluster_from_dcs()
return 'initialized a new cluster'
def handle_starting_instance(self):
@@ -1135,8 +914,6 @@ class Ha(object):
# Check if we are in startup, when paused defer to main loop for manual failovers.
if not self.state_handler.check_for_startup() or self.is_paused():
self.set_start_timeout(None)
if self.is_paused():
self.state_handler.set_state(self.state_handler.is_running() and 'running' or 'stopped')
return None
# state_handler.state == 'starting' here
@@ -1176,16 +953,10 @@ class Ha(object):
def _run_cycle(self):
dcs_failed = False
try:
self.state_handler.reset_cluster_info_state()
self.load_cluster_from_dcs()
if self.is_paused():
self.watchdog.disable()
self._was_paused = True
else:
if self._was_paused:
self.state_handler.schedule_sanity_checks_after_pause()
self._was_paused = False
if not self.cluster.has_member(self.state_handler.name):
self.touch_member()
@@ -1218,9 +989,7 @@ class Ha(object):
# is data directory empty?
if self.state_handler.data_directory_empty():
self.state_handler.set_role('uninitialized')
self.state_handler.stop('immediate')
# In case datadir went away while we were master.
# In case datadir went away while we were master. TODO: check for this and try to stop postgresql.
self.watchdog.disable()
# is this instance the leader?
@@ -1231,20 +1000,9 @@ class Ha(object):
return self.bootstrap() # new node
# "bootstrap", but data directory is not empty
elif not self.sysid_valid(self.cluster.initialize) and self.cluster.is_unlocked() and not self.is_paused():
if not self.state_handler.cb_called and self.state_handler.is_running() \
and not self.state_handler.is_leader():
self._join_aborted = True
logger.error('No initialize key in DCS and PostgreSQL is running as replica, aborting start')
logger.error('Please first start Patroni on the node running as master')
sys.exit(1)
self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=self.state_handler.sysid)
else:
# check if we are allowed to join
data_sysid = self.state_handler.sysid
if not self.sysid_valid(data_sysid):
# data directory is not empty, but no valid sysid, cluster must be broken, suggest reinit
return "data dir for the cluster is not empty, but system ID is invalid; consider doing reinitalize"
if self.sysid_valid(self.cluster.initialize) and self.cluster.initialize != self.state_handler.sysid:
logger.fatal("system ID mismatch, node %s belongs to a different cluster: %s != %s",
self.state_handler.name, self.cluster.initialize, self.state_handler.sysid)
@@ -1256,13 +1014,8 @@ class Ha(object):
self.dcs.delete_leader()
self.dcs.reset_cluster()
return 'removed leader lock because postgres is not running'
# Normally we don't start Postgres in a paused state. We make an exception for the demoted primary
# that needs to be started after it had been stopped by demote. When there is no need to call rewind
# the demote code follows through to starting Postgres right away, however, in the rewind case
# it returns from demote and reaches this point to start PostgreSQL again after rewind. In that
# case it makes no sense to continue to recover() unless rewind has finished successfully.
elif self.state_handler.rewind_failed or not self.state_handler.need_rewind \
or not self.state_handler.can_rewind_or_reinitialize_allowed:
elif not (self.state_handler.rewind_executed or
self.state_handler.need_rewind and self.state_handler.can_rewind):
return 'postgres is not running'
# try to start dead postgres
@@ -1279,11 +1032,11 @@ class Ha(object):
# stops PostgreSQL, therefore, we only reload replication slots if no
# asynchronous processes are running (should be always the case for the master)
if not self._async_executor.busy and not self.state_handler.is_starting():
self.state_handler.sync_replication_slots(self.cluster)
if not self.state_handler.cb_called:
if not self.state_handler.is_leader():
self.state_handler.trigger_check_diverged_lsn()
self.state_handler.call_nowait(ACTION_ON_START)
self.state_handler.sync_replication_slots(self.cluster)
except DCSError:
dcs_failed = True
logger.error('Error communicating with DCS')
@@ -1306,7 +1059,7 @@ class Ha(object):
if self.is_paused():
logger.info('Leader key is not deleted and Postgresql is not stopped due paused state')
self.watchdog.disable()
elif not self._join_aborted:
else:
# FIXME: If stop doesn't reach safepoint quickly enough keepalive is triggered. If shutdown checkpoint
# takes longer than ttl, then leader key is lost and replication might not have sent out all xlog.
# This might not be the desired behavior of users, as a graceful shutdown of the host can mean lost data.
@@ -1314,9 +1067,7 @@ class Ha(object):
disable_wd = self.watchdog.disable if self.watchdog.is_running else None
self.while_not_sync_standby(lambda: self.state_handler.stop(checkpoint=False, on_safepoint=disable_wd))
if not self.state_handler.is_running():
if self.has_lock():
self.dcs.delete_leader()
self.touch_member()
self.dcs.delete_leader()
else:
# XXX: what about when Patroni is started as the wrong user that has access to the watchdog device
# but cannot shut down PostgreSQL. Root would be the obvious example. Would be nice to not kill the
@@ -1340,21 +1091,3 @@ class Ha(object):
no "active" leader watch request in progress.
This usually happens on the master or if the node is running async action"""
self.dcs.event.set()
def get_remote_master(self):
""" In case of standby cluster this will tel us from which remote
master to stream. Config can be both patroni config or
cluster.config.data
"""
cluster_params = self.get_standby_cluster_config()
if cluster_params:
unique_name = 'remote_master:{}'.format(uuid.uuid1())
data = {k: v for k, v in cluster_params.items() if k in RemoteMember.allowed_keys()}
data['no_replication_slot'] = 'primary_slot_name' not in cluster_params
conn_kwargs = {k: cluster_params[k] for k in ('host', 'port') if k in cluster_params}
if conn_kwargs:
data['conn_kwargs'] = conn_kwargs
return RemoteMember(unique_name, data)
-66
View File
@@ -1,66 +0,0 @@
import logging
import os
from copy import deepcopy
from logging.handlers import RotatingFileHandler
from patroni.utils import deep_compare
class PatroniLogger(object):
DEFAULT_LEVEL = 'INFO'
DEFAULT_FORMAT = '%(asctime)s %(levelname)s: %(message)s'
def __init__(self):
self.root_logger = logging.getLogger()
self.config = None
self.handler = None
self.reload_config({'level': 'DEBUG'})
def update_loggers(self):
loggers = deepcopy(self.config.get('loggers') or {})
for name, logger in self.root_logger.manager.loggerDict.items():
if not isinstance(logger, logging.PlaceHolder):
level = loggers.pop(name, logging.NOTSET)
logger.setLevel(level)
for name, level in loggers.items():
logger = self.root_logger.manager.getLogger(name)
logger.setLevel(level)
def reload_config(self, config):
if self.config is None or not deep_compare(self.config, config):
self.root_logger.setLevel(config.get('level', PatroniLogger.DEFAULT_LEVEL))
add_handler = None
if 'dir' in config:
if not isinstance(self.handler, RotatingFileHandler):
add_handler = RotatingFileHandler(os.path.join(config['dir'], __name__))
handler = add_handler or self.handler
handler.maxBytes = int(config.get('file_size', 25000000))
handler.backupCount = int(config.get('file_num', 4))
else:
if self.handler is None or isinstance(self.handler, RotatingFileHandler):
add_handler = logging.StreamHandler()
handler = add_handler or self.handler
oldlogformat = (self.config or {}).get('format', PatroniLogger.DEFAULT_FORMAT)
logformat = config.get('format', PatroniLogger.DEFAULT_FORMAT)
olddateformat = (self.config or {}).get('dateformat') or None
dateformat = config.get('dateformat') or None # Convert empty string to `None`
if oldlogformat != logformat or olddateformat != dateformat or add_handler:
handler.setFormatter(logging.Formatter(logformat, dateformat))
if add_handler:
self.root_logger.addHandler(add_handler)
if self.handler is not None:
self.root_logger.removeHandler(self.handler)
self.handler.close()
self.handler = add_handler
self.config = config.copy()
self.update_loggers()
+394 -646
View File
File diff suppressed because it is too large Load Diff
-171
View File
@@ -1,171 +0,0 @@
import logging
import os
import psutil
import re
import signal
import subprocess
from patroni import call_self
logger = logging.getLogger(__name__)
STOP_SIGNALS = {
'smart': signal.SIGTERM,
'fast': signal.SIGINT,
'immediate': signal.SIGQUIT if os.name != 'nt' else signal.SIGABRT,
}
class PostmasterProcess(psutil.Process):
def __init__(self, pid):
self.is_single_user = False
if pid < 0:
pid = -pid
self.is_single_user = True
super(PostmasterProcess, self).__init__(pid)
@staticmethod
def _read_postmaster_pidfile(data_dir):
"""Reads and parses postmaster.pid from the data directory
:returns dictionary of values if successful, empty dictionary otherwise
"""
pid_line_names = ['pid', 'data_dir', 'start_time', 'port', 'socket_dir', 'listen_addr', 'shmem_key']
try:
with open(os.path.join(data_dir, 'postmaster.pid')) as f:
return {name: line.rstrip('\n') for name, line in zip(pid_line_names, f)}
except IOError:
return {}
def _is_postmaster_process(self):
try:
start_time = int(self._postmaster_pid.get('start_time', 0))
if start_time and abs(self.create_time() - start_time) > 3:
logger.info('Process %s is not postmaster, too much difference between PID file start time %s and '
'process start time %s', self.pid, self.create_time(), start_time)
return False
except ValueError:
logger.warning('Garbage start time value in pid file: %r', self._postmaster_pid.get('start_time'))
# Extra safety check. The process can't be ourselves, our parent or our direct child.
if self.pid == os.getpid() or self.pid == os.getppid() or self.ppid() == os.getpid():
logger.info('Patroni (pid=%s, ppid=%s), "fake postmaster" (pid=%s, ppid=%s)',
os.getpid(), os.getppid(), self.pid, self.ppid())
return False
return True
@classmethod
def _from_pidfile(cls, data_dir):
postmaster_pid = PostmasterProcess._read_postmaster_pidfile(data_dir)
try:
pid = int(postmaster_pid.get('pid', 0))
if pid:
proc = cls(pid)
proc._postmaster_pid = postmaster_pid
return proc
except ValueError:
pass
@staticmethod
def from_pidfile(data_dir):
try:
proc = PostmasterProcess._from_pidfile(data_dir)
return proc if proc and proc._is_postmaster_process() else None
except psutil.NoSuchProcess:
return None
@classmethod
def from_pid(cls, pid):
try:
return cls(pid)
except psutil.NoSuchProcess:
return None
def signal_stop(self, mode):
"""Signal postmaster process to stop
:returns None if signaled, True if process is already gone, False if error
"""
if self.is_single_user:
logger.warning("Cannot stop server; single-user server is running (PID: {0})".format(self.pid))
return False
try:
self.send_signal(STOP_SIGNALS[mode])
except psutil.NoSuchProcess:
return True
except psutil.AccessDenied as e:
logger.warning("Could not send stop signal to PostgreSQL (error: {0})".format(e))
return False
return None
def wait_for_user_backends_to_close(self):
# These regexps are cross checked against versions PostgreSQL 9.1 .. 11
aux_proc_re = re.compile("(?:postgres:)( .*:)? (?:(?:archiver|startup|autovacuum launcher|autovacuum worker|"
"checkpointer|logger|stats collector|wal receiver|wal writer|writer)(?: process )?|"
"walreceiver|wal sender process|walsender|walwriter|background writer|"
"logical replication launcher|logical replication worker for|bgworker:) ")
try:
children = self.children()
except psutil.Error:
return logger.debug('Failed to get list of postmaster children')
user_backends = []
user_backends_cmdlines = []
for child in children:
try:
cmdline = child.cmdline()[0]
if not aux_proc_re.match(cmdline):
user_backends.append(child)
user_backends_cmdlines.append(cmdline)
except psutil.NoSuchProcess:
pass
if user_backends:
logger.debug('Waiting for user backends %s to close', ', '.join(user_backends_cmdlines))
psutil.wait_procs(user_backends)
logger.debug("Backends closed")
@staticmethod
def start(pgcommand, data_dir, conf, options):
# Unfortunately `pg_ctl start` does not return postmaster pid to us. Without this information
# it is hard to know the current state of postgres startup, so we had to reimplement pg_ctl start
# in python. It will start postgres, wait for port to be open and wait until postgres will start
# accepting connections.
# Important!!! We can't just start postgres using subprocess.Popen, because in this case it
# will be our child for the rest of our live and we will have to take care of it (`waitpid`).
# So we will use the same approach as pg_ctl uses: start a new process, which will start postgres.
# This process will write postmaster pid to stdout and exit immediately. Now it's responsibility
# of init process to take care about postmaster.
# In order to make everything portable we can't use fork&exec approach here, so we will call
# ourselves and pass list of arguments which must be used to start postgres.
# On Windows, in order to run a side-by-side assembly the specified env must include a valid SYSTEMROOT.
env = {p: os.environ[p] for p in ('PATH', 'LD_LIBRARY_PATH', 'LC_ALL', 'LANG', 'SYSTEMROOT') if p in os.environ}
try:
proc = PostmasterProcess._from_pidfile(data_dir)
if proc and not proc._is_postmaster_process():
# Upon start postmaster process performs various safety checks if there is a postmaster.pid
# file in the data directory. Although Patroni already detected that the running process
# corresponding to the postmaster.pid is not a postmaster, the new postmaster might fail
# to start, because it thinks that postmaster.pid is already locked.
# Important!!! Unlink of postmaster.pid isn't an option, because it has a lot of nasty race conditions.
# Luckily there is a workaround to this problem, we can pass the pid from postmaster.pid
# in the `PG_GRANDPARENT_PID` environment variable and postmaster will ignore it.
logger.info("Telling pg_ctl that it is safe to ignore postmaster.pid for process %s", proc.pid)
env['PG_GRANDPARENT_PID'] = str(proc.pid)
except psutil.NoSuchProcess:
pass
cmdline = [pgcommand, '-D', data_dir, '--config-file={}'.format(conf)] + options
logger.debug("Starting postgres: %s", " ".join(cmdline))
proc = call_self(['pg_ctl_start'] + cmdline, close_fds=(os.name != 'nt'),
stdout=subprocess.PIPE, env=env)
pid = int(proc.stdout.readline().strip())
proc.wait()
logger.info('postmaster pid=%s', pid)
# TODO: In an extremely unlikely case, the process could have exited and the pid reassigned. The start
# initiation time is not accurate enough to compare to create time as start time would also likely
# be relatively close. We need the subprocess extract pid+start_time in a race free manner.
return PostmasterProcess.from_pid(pid)
+7 -6
View File
@@ -224,12 +224,13 @@ class WALERestore(object):
lsn_name = 'location'
con.autocommit = True
with con.cursor() as cur:
cur.execute(("SELECT CASE WHEN pg_catalog.pg_is_in_recovery()"
" THEN GREATEST(pg_catalog.pg_{0}_{1}_diff(COALESCE("
"pg_last_{0}_receive_{1}(), '0/0'), %s)::bigint, "
"pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), %s)::bigint)"
" ELSE pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}_{1}(), %s)::bigint"
" END").format(wal_name, lsn_name),
cur.execute("""SELECT CASE WHEN pg_is_in_recovery()
THEN GREATEST(
pg_{0}_{1}_diff(COALESCE(
pg_last_{0}_receive_{1}(), '0/0'), %s)::bigint,
pg_{0}_{1}_diff(pg_last_{0}_replay_{1}(), %s)::bigint)
ELSE pg_{0}_{1}_diff(pg_current_{0}_{1}(), %s)::bigint
END""".format(wal_name, lsn_name),
(backup_start_lsn, backup_start_lsn, backup_start_lsn))
diff_in_bytes = int(cur.fetchone()[0])
+14 -9
View File
@@ -1,5 +1,7 @@
import contextlib
import random
import time
import re
from dateutil import tz
from patroni.exceptions import PatroniException
@@ -94,17 +96,17 @@ def strtol(value, strict=True):
True
"""
value = str(value).strip()
ln = len(value)
l = len(value)
i = 0
# skip sign:
if i < ln and value[i] in ('-', '+'):
if i < l and value[i] in ('-', '+'):
i += 1
# we always expect to get digit in the beginning
if i < ln and value[i].isdigit():
if i < l and value[i].isdigit():
if value[i] == '0':
i += 1
if i < ln and value[i] in ('x', 'X'): # '0' followed by 'x': HEX
if i < l and value[i] in ('x', 'X'): # '0' followed by 'x': HEX
base = 16
i += 1
else: # just starts with '0': OCT
@@ -113,7 +115,7 @@ def strtol(value, strict=True):
base = 10
ret = None
while i <= ln:
while i <= l:
try: # try to find maximally long number
i += 1 # by giving to `int` longer and longer strings
ret = int(value[:i], base)
@@ -193,6 +195,10 @@ def _sleep(interval):
time.sleep(interval)
def is_valid_pg_version(version):
return re.match(r'[1-9][0-9]?(\.(0|([1-9][0-9]?))){2}$', version)
class RetryFailedError(PatroniException):
"""Raised when retrying an operation ultimately failed, after retrying the maximum number of attempts."""
@@ -277,7 +283,6 @@ def polling_loop(timeout, interval=1):
time.sleep(interval)
def split_host_port(value, default_port):
t = value.rsplit(':', 1)
t.append(default_port)
return t[0], int(t[1])
@contextlib.contextmanager
def null_context():
yield
+1 -1
View File
@@ -1 +1 @@
__version__ = '1.5.5'
__version__ = '1.3'
+3 -3
View File
@@ -133,7 +133,6 @@ class Watchdog(object):
try:
self.impl.open()
actual_timeout = self._set_timeout()
except WatchdogError as e:
logger.warning("Could not activate %s: %s", self.impl.describe(), e)
self.impl = NullWatchdog()
@@ -142,6 +141,8 @@ class Watchdog(object):
logger.warning("Watchdog implementation can't be disabled."
" Watchdog will trigger after Patroni loses leader key.")
actual_timeout = self._set_timeout()
if not self.impl.is_running or actual_timeout > self.config.timeout:
if self.config.mode == MODE_REQUIRED:
if self.impl.is_null:
@@ -201,8 +202,7 @@ class Watchdog(object):
@synchronized
def keepalive(self):
try:
if self.active:
self.impl.keepalive()
self.impl.keepalive()
# In case there are any pending configuration changes apply them now.
if self.active and self.config != self.active_config:
if self.config.mode != MODE_OFF and self.active_config.mode == MODE_OFF:
+10 -20
View File
@@ -155,25 +155,21 @@ class LinuxWatchdogDevice(WatchdogBase):
def can_be_disabled(self):
return self.get_support().has_MAGICCLOSE
def _ioctl(self, func, arg):
"""Runs the specified ioctl on the underlying fd.
Raises WatchdogError if the device is closed.
Raises OSError or IOError (Python 2) when the ioctl fails."""
def _ioctl(self, func, arg, mutate_arg=False):
if self._fd is None:
raise WatchdogError("Watchdog device is closed")
fcntl.ioctl(self._fd, func, arg, True)
result = fcntl.ioctl(self._fd, func, arg, mutate_arg)
if result < 0:
raise IOError(result)
def get_support(self):
if self._support_cache is None:
info = watchdog_info()
try:
self._ioctl(WDIOC_GETSUPPORT, info)
except (WatchdogError, OSError, IOError) as e:
raise WatchdogError("Could not get information about watchdog device: {}".format(e))
self._ioctl(WDIOC_GETSUPPORT, info, True)
self._support_cache = WatchdogInfo(info.options,
info.firmware_version,
bytearray(info.identity).decode(errors='ignore').rstrip('\x00'))
str(bytearray(info.identity)).rstrip('\x00'))
return self._support_cache
def describe(self):
@@ -184,7 +180,7 @@ class LinuxWatchdogDevice(WatchdogBase):
try:
_, version, identity = self.get_support()
ver_str = " (firmware {0})".format(version) if version else ""
except WatchdogError:
except WatchdogError: # XXX: Can it really be raise when self._fd is not None?
pass
return identity + ver_str + dev_str
@@ -203,17 +199,11 @@ class LinuxWatchdogDevice(WatchdogBase):
timeout = int(timeout)
if not 0 < timeout < 0xFFFF:
raise WatchdogError("Invalid timeout {0}. Supported values are between 1 and 65535".format(timeout))
try:
self._ioctl(WDIOC_SETTIMEOUT, ctypes.c_int(timeout))
except (WatchdogError, OSError, IOError) as e:
raise WatchdogError("Could not set timeout on watchdog device: {}".format(e))
self._ioctl(WDIOC_SETTIMEOUT, ctypes.c_int(timeout))
def get_timeout(self):
timeout = ctypes.c_int()
try:
self._ioctl(WDIOC_GETTIMEOUT, timeout)
except (WatchdogError, OSError, IOError) as e:
raise WatchdogError("Could not get timeout on watchdog device: {}".format(e))
self._ioctl(WDIOC_GETTIMEOUT, timeout, True)
return timeout.value
+2 -11
View File
@@ -11,11 +11,6 @@ restapi:
# username: username
# password: password
# ctl:
# insecure: false # Allow connections to SSL sites without certs
# certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
# cacert: /etc/ssl/certs/ssl-cacert-snakeoil.pem
etcd:
host: 127.0.0.1:2379
@@ -29,10 +24,6 @@ bootstrap:
maximum_lag_on_failover: 1048576
# master_start_timeout: 300
# synchronous_mode: false
#standby_cluster:
#host: 127.0.0.1
#port: 1111
#primary_slot_name: patroni
postgresql:
use_pg_rewind: true
# use_slots: true
@@ -40,8 +31,8 @@ bootstrap:
# wal_level: hot_standby
# hot_standby: "on"
# wal_keep_segments: 8
# max_wal_senders: 10
# max_replication_slots: 10
# max_wal_senders: 5
# max_replication_slots: 5
# wal_log_hints: "on"
# archive_mode: "on"
# archive_timeout: 1800s
+2 -10
View File
@@ -11,11 +11,6 @@ restapi:
# username: username
# password: password
# ctl:
# insecure: false # Allow connections to SSL sites without certs
# certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
# cacert: /etc/ssl/certs/ssl-cacert-snakeoil.pem
etcd:
host: 127.0.0.1:2379
@@ -34,8 +29,8 @@ bootstrap:
# wal_level: hot_standby
# hot_standby: "on"
# wal_keep_segments: 8
# max_wal_senders: 10
# max_replication_slots: 10
# max_wal_senders: 5
# max_replication_slots: 5
# wal_log_hints: "on"
# archive_mode: "on"
# archive_timeout: 1800s
@@ -80,9 +75,6 @@ postgresql:
password: zalando
parameters:
unix_socket_directories: '.'
basebackup:
- verbose
- max-rate: 100M
tags:
nofailover: false
noloadbalance: false
+2 -7
View File
@@ -11,11 +11,6 @@ restapi:
username: username
password: password
# ctl:
# insecure: false # Allow connections to SSL sites without certs
# certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
# cacert: /etc/ssl/certs/ssl-cacert-snakeoil.pem
etcd:
host: 127.0.0.1:2379
@@ -34,8 +29,8 @@ bootstrap:
# wal_level: hot_standby
# hot_standby: "on"
# wal_keep_segments: 8
# max_wal_senders: 10
# max_replication_slots: 10
# max_wal_senders: 5
# max_replication_slots: 5
# wal_log_hints: "on"
# archive_mode: "on"
# archive_timeout: 1800s
+5 -6
View File
@@ -1,16 +1,15 @@
urllib3>=1.19.1,!=1.21
urllib3>=1.9
boto
psycopg2>=2.5.4
psycopg2>=2.6.1
PyYAML
requests
six >= 1.7
kazoo>=1.3.1
kazoo==2.2.1
python-etcd>=0.4.3,<0.5
python-consul>=0.7.0
python-consul==0.7.0
click>=4.1
prettytable>=0.7
tzlocal
python-dateutil
psutil>=2.0.0
psutil
cdiff
kubernetes>=2.0.0,<=7.0.0,!=4.0.*,!=5.0.*
+11 -30
View File
@@ -32,10 +32,9 @@ VERSION = read_version(MAIN_PACKAGE)
DESCRIPTION = 'PostgreSQL High-Available orchestrator and CLI'
LICENSE = 'The MIT License'
URL = 'https://github.com/zalando/patroni'
AUTHOR = 'Alexander Kukushkin, Dmitrii Dolgov, Oleksii Kliukin'
AUTHOR_EMAIL = '[email protected], [email protected], [email protected]'
KEYWORDS = 'etcd governor patroni postgresql postgres ha haproxy confd' +\
' zookeeper exhibitor consul streaming replication kubernetes k8s'
AUTHOR = 'Alexander Kukushkin, Oleksii Kliukin, Feike Steenbergen'
AUTHOR_EMAIL = '[email protected], [email protected], [email protected]'
KEYWORDS = 'etcd governor patroni postgresql postgres ha haproxy confd zookeeper exhibitor consul streaming replication'
COVERAGE_XML = True
COVERAGE_HTML = False
@@ -44,22 +43,16 @@ JUNIT_XML = True
# Add here all kinds of additional classifiers as defined under
# https://pypi.python.org/pypi?%3Aaction=list_classifiers
CLASSIFIERS = [
'Development Status :: 5 - Production/Stable',
'Development Status :: 4 - Beta',
'Environment :: Console',
'Intended Audience :: Developers',
'Intended Audience :: System Administrators',
'License :: OSI Approved :: MIT License',
'Operating System :: MacOS',
'Operating System :: POSIX :: Linux',
'Operating System :: POSIX :: BSD :: FreeBSD',
'Operating System :: Microsoft :: Windows',
'Programming Language :: Python',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: Implementation :: CPython',
]
@@ -94,7 +87,7 @@ class PyTest(TestCommand):
def run_tests(self):
try:
import pytest
except Exception:
except:
raise RuntimeError('py.test is not installed, run: pip install pytest')
params = {'args': self.test_args}
if self.cov:
@@ -107,8 +100,6 @@ class PyTest(TestCommand):
silence = logging.WARNING
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=os.getenv('LOGLEVEL', silence))
params['args'] += ['-s' if logging.getLogger().getEffectiveLevel() < silence else '--capture=fd']
if not os.getenv('SYSTEMROOT'):
os.environ['SYSTEMROOT'] = '/'
errno = pytest.main(**params)
sys.exit(errno)
@@ -124,23 +115,13 @@ def read(fname):
def setup_package():
# Assemble additional setup commands
cmdclass = {'test': PyTest}
cmdclass = {}
cmdclass['test'] = PyTest
# Some helper variables
version = os.getenv('GO_PIPELINE_LABEL', VERSION)
install_requires = []
extras_require = {'aws': ['boto'], 'etcd': ['python-etcd'], 'consul': ['python-consul'],
'exhibitor': ['kazoo'], 'zookeeper': ['kazoo'], 'kubernetes': ['kubernetes']}
for r in get_install_requirements('requirements.txt'):
extra = False
for e, v in extras_require.items():
if r.startswith(v[0]):
extras_require[e] = [r]
extra = True
if not extra:
install_requires.append(r)
install_reqs = get_install_requirements('requirements.txt')
command_options = {'test': {'test_suite': ('setup.py', 'tests')}}
if JUNIT_XML:
@@ -164,10 +145,10 @@ def setup_package():
test_suite='tests',
packages=find_packages(exclude=['tests', 'tests.*']),
package_data={MAIN_PACKAGE: ["*.json"]},
install_requires=install_requires,
extras_require=extras_require,
install_requires=install_reqs,
setup_requires=['flake8'],
cmdclass=cmdclass,
tests_require=['flake8', 'mock>=2.0.0', 'pytest-cov', 'pytest'],
tests_require=['mock>=2.0.0', 'pytest-cov', 'pytest'],
command_options=command_options,
entry_points={'console_scripts': CONSOLE_SCRIPTS},
)
+15 -77
View File
@@ -3,10 +3,9 @@ import json
import psycopg2
import unittest
from mock import Mock, PropertyMock, patch
from mock import Mock, patch
from patroni.api import RestApiHandler, RestApiServer
from patroni.dcs import ClusterConfig, Member
from patroni.ha import _MemberStatus
from patroni.utils import tzutc
from six import BytesIO as IO
from six.moves import BaseHTTPServer
@@ -37,13 +36,9 @@ class MockPostgresql(object):
def postmaster_start_time():
return str(postmaster_start_time)
@staticmethod
def replica_cached_timeline(_):
return 2
class MockWatchdog(object):
is_healthy = False
is_healthy = True
class MockHa(object):
@@ -52,11 +47,7 @@ class MockHa(object):
watchdog = MockWatchdog()
@staticmethod
def is_leader():
return False
@staticmethod
def reinitialize(_):
def reinitialize():
return 'reinitialize'
@staticmethod
@@ -73,7 +64,7 @@ class MockHa(object):
@staticmethod
def fetch_nodes_statuses(members):
return [_MemberStatus(None, True, None, 0, None, {}, False)]
return [[None, True, None, None, {}]]
@staticmethod
def schedule_future_restart(data):
@@ -91,14 +82,6 @@ class MockHa(object):
def wakeup():
pass
@staticmethod
def is_paused():
return True
@staticmethod
def is_standby_cluster():
return False
class MockPatroni(object):
@@ -108,7 +91,7 @@ class MockPatroni(object):
dcs = Mock()
tags = {}
version = '0.00'
noloadbalance = PropertyMock(return_value=False)
noloadbalance = Mock(return_value=False)
scheduled_restart = {'schedule': future_restart_time,
'postmaster_start_time': postgresql.postmaster_start_time()}
@@ -131,14 +114,16 @@ class MockRequest(object):
class MockRestApiServer(RestApiServer):
def __init__(self, Handler, request, config=None):
def __init__(self, Handler, request):
self.socket = 0
self.serve_forever = Mock()
BaseHTTPServer.HTTPServer.__init__ = Mock()
MockRestApiServer._BaseServer__is_shut_down = Mock()
MockRestApiServer._BaseServer__shutdown_request = True
config = config or {'listen': '127.0.0.1:8008', 'auth': 'test:test', 'certfile': 'dumb'}
config = {'listen': '127.0.0.1:8008', 'auth': 'test:test'}
super(MockRestApiServer, self).__init__(MockPatroni(), config)
config['certfile'] = 'dumb'
self.reload_config(config)
Handler(MockRequest(request), ('0.0.0.0', 8080), self)
@@ -154,14 +139,7 @@ class TestRestApiHandler(unittest.TestCase):
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'master'})):
MockRestApiServer(RestApiHandler, 'GET /replica')
MockRestApiServer(RestApiHandler, 'GET /master')
MockPatroni.dcs.cluster.sync.sync_standby = MockPostgresql.name
MockPatroni.dcs.cluster.is_synchronous_mode = Mock(return_value=True)
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'replica'})):
MockRestApiServer(RestApiHandler, 'GET /synchronous')
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'replica'})):
MockPatroni.dcs.cluster.sync.sync_standby = ''
MockRestApiServer(RestApiHandler, 'GET /asynchronous')
MockPatroni.ha.is_leader = Mock(return_value=True)
MockPatroni.dcs.cluster.leader.name = MockPostgresql.name
MockRestApiServer(RestApiHandler, 'GET /replica')
MockPatroni.dcs.cluster = None
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'master'})):
@@ -169,15 +147,10 @@ class TestRestApiHandler(unittest.TestCase):
with patch.object(MockHa, 'restart_scheduled', Mock(return_value=True)):
MockRestApiServer(RestApiHandler, 'GET /master')
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /master'))
with patch.object(RestApiServer, 'query', Mock(return_value=[('', 1, '', '', '', '', False, '')])):
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
with patch.object(MockHa, 'is_standby_cluster', Mock(return_value=True)):
MockRestApiServer(RestApiHandler, 'GET /standby_leader')
def test_do_OPTIONS(self):
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'OPTIONS / HTTP/1.0'))
@patch.object(MockPostgresql, 'state', PropertyMock(return_value='stopped'))
def test_do_GET_patroni(self):
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
@@ -289,7 +262,7 @@ class TestRestApiHandler(unittest.TestCase):
def test_do_POST_reinitialize(self, mock_dcs):
cluster = mock_dcs.get_cluster.return_value
cluster.is_paused.return_value = False
request = 'POST /reinitialize HTTP/1.0' + self._authorization + '\nContent-Length: 15\n\n{"force": true}'
request = 'POST /reinitialize HTTP/1.0' + self._authorization
MockRestApiServer(RestApiHandler, request)
with patch.object(MockHa, 'reinitialize', Mock(return_value=None)):
MockRestApiServer(RestApiHandler, request)
@@ -303,13 +276,11 @@ class TestRestApiHandler(unittest.TestCase):
@patch('time.sleep', Mock())
@patch.object(MockPatroni, 'dcs')
def test_do_POST_switchover(self, dcs):
def test_do_POST_failover(self, dcs):
dcs.loop_wait = 10
cluster = dcs.get_cluster.return_value
cluster.is_synchronous_mode.return_value = False
cluster.is_paused.return_value = False
post = 'POST /switchover HTTP/1.0' + self._authorization + '\nContent-Length: '
post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}')
@@ -319,22 +290,14 @@ class TestRestApiHandler(unittest.TestCase):
cluster.leader.name = 'postgresql1'
MockRestApiServer(RestApiHandler, request)
request = post + '25\n\n{"leader": "postgresql1"}'
cluster.is_paused.return_value = True
MockRestApiServer(RestApiHandler, request)
cluster.is_paused.return_value = False
for cluster.is_synchronous_mode.return_value in (True, False):
MockRestApiServer(RestApiHandler, request)
MockRestApiServer(RestApiHandler, post + '25\n\n{"leader": "postgresql1"}')
cluster.leader.name = 'postgresql2'
request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
MockRestApiServer(RestApiHandler, request)
cluster.leader.name = 'postgresql1'
for cluster.is_synchronous_mode.return_value in (True, False):
MockRestApiServer(RestApiHandler, request)
MockRestApiServer(RestApiHandler, request)
cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
Member(0, 'postgresql2', 30, {'api_url': 'http'})]
@@ -348,7 +311,6 @@ class TestRestApiHandler(unittest.TestCase):
cluster2 = cluster.copy()
cluster2.leader.name = 'postgresql0'
cluster2.is_unlocked.return_value = False
dcs.get_cluster.side_effect = [cluster, cluster2]
MockRestApiServer(RestApiHandler, request)
@@ -383,27 +345,3 @@ class TestRestApiHandler(unittest.TestCase):
# Invalid date
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}'))
@patch.object(MockPatroni, 'dcs', Mock())
def test_do_POST_failover(self):
post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}')
MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}')
@patch('ssl.wrap_socket', Mock(return_value=0))
class TestRestApiServer(unittest.TestCase):
def test_reload_config(self):
bad_config = {'listen': 'foo'}
self.assertRaises(ValueError, MockRestApiServer, None, '', bad_config)
srv = MockRestApiServer(lambda a1, a2, a3: None, '')
self.assertRaises(ValueError, srv.reload_config, bad_config)
self.assertRaises(ValueError, srv.reload_config, {})
srv.reload_config({'listen': '127.0.0.2:8008'})
def test_handle_error(self):
try:
raise Exception()
except Exception:
self.assertIsNone(MockRestApiServer.handle_error(None, ('127.0.0.1', 55555)))
+1 -7
View File
@@ -8,7 +8,7 @@ from threading import Thread
class TestAsyncExecutor(unittest.TestCase):
def setUp(self):
self.a = AsyncExecutor(Mock(), Mock())
self.a = AsyncExecutor(Mock())
@patch.object(Thread, 'start', Mock())
def test_run_async(self):
@@ -17,12 +17,6 @@ class TestAsyncExecutor(unittest.TestCase):
def test_run(self):
self.a.run(Mock(side_effect=Exception()))
def test_cancel(self):
self.a.cancel()
self.a.schedule('foo')
self.a.cancel()
self.a.run(Mock())
class TestCriticalTask(unittest.TestCase):
+3 -22
View File
@@ -1,6 +1,6 @@
import os
import sys
import unittest
import sys
from mock import MagicMock, Mock, patch
from patroni.config import Config
@@ -23,15 +23,13 @@ class TestConfig(unittest.TestCase):
def test_set_dynamic_configuration(self):
with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
self.assertIsNone(self.config.set_dynamic_configuration({'foo': 'bar'}))
self.assertTrue(self.config.set_dynamic_configuration({'synchronous_mode': True, 'standby_cluster': {}}))
self.assertTrue(self.config.set_dynamic_configuration({'synchronous_mode': True}))
def test_reload_local_configuration(self):
os.environ.update({
'PATRONI_NAME': 'postgres0',
'PATRONI_NAMESPACE': '/patroni/',
'PATRONI_SCOPE': 'batman2',
'PATRONI_LOGLEVEL': 'ERROR',
'PATRONI_LOG_LOGGERS': 'patroni.postmaster: WARNING, urllib3: DEBUG',
'PATRONI_RESTAPI_USERNAME': 'username',
'PATRONI_RESTAPI_PASSWORD': 'password',
'PATRONI_RESTAPI_LISTEN': '0.0.0.0:8008',
@@ -51,10 +49,6 @@ class TestConfig(unittest.TestCase):
'PATRONI_ETCD_CERT': '/cert',
'PATRONI_ETCD_KEY': '/key',
'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
'PATRONI_CONSUL_REGISTER_SERVICE': 'on',
'PATRONI_KUBERNETES_LABELS': 'a:b:c',
'PATRONI_KUBERNETES_SCOPE_LABEL': 'a',
'PATRONI_KUBERNETES_PORTS': '[{"name": "postgresql"}]',
'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',
'PATRONI_EXHIBITOR_PORT': '8181',
@@ -73,13 +67,12 @@ class TestConfig(unittest.TestCase):
self.assertRaises(Exception, config.reload_local_configuration, True)
self.assertTrue(config.reload_local_configuration(True))
self.assertTrue(config.reload_local_configuration())
self.assertIsNone(config.reload_local_configuration())
@patch('tempfile.mkstemp', Mock(return_value=[3000, 'blabla']))
@patch('os.path.exists', Mock(return_value=True))
@patch('os.remove', Mock(side_effect=IOError))
@patch('os.close', Mock(side_effect=IOError))
@patch('shutil.move', Mock(return_value=None))
@patch('os.rename', Mock(return_value=None))
@patch('json.dump', Mock())
def test_save_cache(self):
self.config.set_dynamic_configuration({'ttl': 30, 'postgresql': {'foo': 'bar'}})
@@ -87,15 +80,3 @@ class TestConfig(unittest.TestCase):
self.config.save_cache()
with patch('os.fdopen', MagicMock()):
self.config.save_cache()
def test_standby_cluster_parameters(self):
dynamic_configuration = {
'standby_cluster': {
'create_replica_methods': ['wal_e', 'basebackup'],
'host': 'localhost',
'port': 5432
}
}
self.config.set_dynamic_configuration(dynamic_configuration)
for name, value in dynamic_configuration['standby_cluster'].items():
self.assertEqual(self.config['standby_cluster'][name], value)
+14 -49
View File
@@ -3,8 +3,7 @@ import unittest
from consul import ConsulException, NotFound
from mock import Mock, patch
from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulInternalError, \
ConsulError, HTTPClient, InvalidSessionTTL, InvalidSession
from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulInternalError, ConsulError, HTTPClient
from test_etcd import SleepException
@@ -46,14 +45,9 @@ class TestHTTPClient(unittest.TestCase):
def test_get(self):
self.client.get(Mock(), '')
self.client.get(Mock(), '', {'wait': '1s', 'index': 1, 'token': 'foo'})
self.client.get(Mock(), '', {'wait': '1s', 'index': 1})
self.client.http.request.return_value.status = 500
self.client.http.request.return_value.data = b'Foo'
self.assertRaises(ConsulInternalError, self.client.get, Mock(), '')
self.client.http.request.return_value.data = b"Invalid Session TTL '3000000000', must be between [10s=24h0m0s]"
self.assertRaises(InvalidSessionTTL, self.client.get, Mock(), '')
self.client.http.request.return_value.data = b"invalid session '16492f43-c2d6-5307-432f-e32d6f7bcbd0'"
self.assertRaises(InvalidSession, self.client.get, Mock(), '')
def test_unknown_method(self):
try:
@@ -64,7 +58,7 @@ class TestHTTPClient(unittest.TestCase):
def test_put(self):
self.client.put(Mock(), '/v1/session/create')
self.client.put(Mock(), '/v1/session/create', params=[], data='{"foo": "bar"}')
self.client.put(Mock(), '/v1/session/create', data='{"foo": "bar"}')
@patch.object(consul.Consul.KV, 'get', kv_get)
@@ -75,13 +69,7 @@ class TestConsul(unittest.TestCase):
@patch.object(consul.Consul.KV, 'get', kv_get)
@patch.object(consul.Consul.KV, 'delete', Mock())
def setUp(self):
Consul({'ttl': 30, 'scope': 't', 'name': 'p', 'url': 'https://l:1', 'retry_timeout': 10,
'verify': 'on', 'key': 'foo', 'cert': 'bar', 'cacert': 'buz', 'token': 'asd', 'dc': 'dc1',
'register_service': True})
Consul({'ttl': 30, 'scope': 't_', 'name': 'p', 'url': 'https://l:1', 'retry_timeout': 10,
'verify': 'on', 'cert': 'bar', 'cacert': 'buz', 'register_service': True})
self.c = Consul({'ttl': 30, 'scope': 'test', 'name': 'postgresql1', 'host': 'localhost:1', 'retry_timeout': 10,
'register_service': True})
self.c = Consul({'ttl': 30, 'scope': 'test', 'name': 'postgresql1', 'host': 'localhost:1', 'retry_timeout': 10})
self.c._base_path = '/service/good'
self.c._load_cluster()
@@ -92,9 +80,7 @@ class TestConsul(unittest.TestCase):
self.assertRaises(SleepException, self.c.create_session)
@patch.object(consul.Consul.Session, 'renew', Mock(side_effect=NotFound))
@patch.object(consul.Consul.Session, 'create', Mock(side_effect=[InvalidSessionTTL, ConsulException]))
@patch.object(consul.Consul.Agent, 'self', Mock(return_value={'Config': {'SessionTTLMin': 0}}))
@patch.object(HTTPClient, 'set_ttl', Mock(side_effect=ValueError))
@patch.object(consul.Consul.Session, 'create', Mock(side_effect=ConsulException))
def test_referesh_session(self):
self.c._session = '1'
self.assertFalse(self.c.refresh_session())
@@ -112,18 +98,17 @@ class TestConsul(unittest.TestCase):
self.c._session = 'fd4f44fe-2cac-bba5-a60b-304b51ff39b8'
self.assertIsInstance(self.c.get_cluster(), Cluster)
@patch.object(consul.Consul.KV, 'delete', Mock(side_effect=[ConsulException, True, True, True]))
@patch.object(consul.Consul.KV, 'put', Mock(side_effect=[True, ConsulException, InvalidSession]))
@patch.object(consul.Consul.KV, 'delete', Mock(side_effect=[ConsulException, True, True]))
@patch.object(consul.Consul.KV, 'put', Mock(side_effect=[True, ConsulException]))
def test_touch_member(self):
self.c.refresh_session = Mock(return_value=False)
self.c.touch_member({'conn_url': 'postgres://replicator:[email protected]:5433/postgres',
'api_url': 'http://127.0.0.1:8009/patroni'})
self.c._register_service = True
self.c.refresh_session = Mock(return_value=True)
for _ in range(0, 4):
self.c.touch_member({'balbla': 'blabla'})
self.c.touch_member('balbla')
self.c.touch_member('balbla')
self.c.touch_member('balbla')
self.c.refresh_session = Mock(return_value=False)
self.c.touch_member('balbla')
@patch.object(consul.Consul.KV, 'put', Mock(side_effect=InvalidSession))
@patch.object(consul.Consul.KV, 'put', Mock(return_value=False))
def test_take_leader(self):
self.c.set_ttl(20)
self.c.refresh_session = Mock()
@@ -143,7 +128,7 @@ class TestConsul(unittest.TestCase):
@patch.object(consul.Consul.Session, 'renew', Mock())
def test_update_leader(self):
self.c.update_leader(None)
self.c.update_leader()
@patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
def test_delete_leader(self):
@@ -177,23 +162,3 @@ class TestConsul(unittest.TestCase):
def test_sync_state(self):
self.assertTrue(self.c.set_sync_state_value('{}'))
self.assertTrue(self.c.delete_sync_state())
@patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
def test_set_history_value(self):
self.assertTrue(self.c.set_history_value('{}'))
@patch.object(consul.Consul.Agent.Service, 'register', Mock(side_effect=(False, True)))
@patch.object(consul.Consul.Agent.Service, 'deregister', Mock(return_value=True))
def test_update_service(self):
d = {'role': 'replica', 'api_url': 'http://a/t', 'conn_url': 'pg://c:1', 'state': 'running'}
self.assertIsNone(self.c.update_service({}, {}))
self.assertFalse(self.c.update_service({}, d))
self.assertTrue(self.c.update_service(d, d))
self.assertIsNone(self.c.update_service(d, d))
d['state'] = 'stopped'
self.assertTrue(self.c.update_service(d, d, force=True))
d['state'] = 'unknown'
self.assertIsNone(self.c.update_service({}, d))
d['state'] = 'running'
d['role'] = 'bla'
self.assertIsNone(self.c.update_service({}, d))
+95 -163
View File
@@ -5,17 +5,15 @@ import sys
import unittest
from click.testing import CliRunner
from datetime import datetime, timedelta
from mock import patch, Mock
from patroni.ctl import ctl, store_config, load_config, output_members, request_patroni, get_dcs, parse_dcs, \
from patroni.ctl import ctl, members, store_config, load_config, output_members, request_patroni, get_dcs, parse_dcs, \
get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException, apply_config_changes, \
format_config_for_editing, show_diff, invoke_editor, format_pg_version
from patroni.dcs.etcd import Client, Failover
from patroni.utils import tzutc
format_config_for_editing, show_diff, invoke_editor
from patroni.dcs.etcd import Client
from psycopg2 import OperationalError
from test_etcd import etcd_read, requests_get, socket_getaddrinfo, MockResponse
from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \
get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader, get_cluster, Member
get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader
from test_postgresql import MockConnect, psycopg2_connect
CONFIG_FILE_PATH = './test-ctl.yaml'
@@ -33,7 +31,7 @@ def test_rw_config():
@patch('patroni.ctl.load_config',
Mock(return_value={'scope': 'alpha', 'postgresql': {'data_dir': '.', 'parameters': {}, 'retry_timeout': 5},
Mock(return_value={'postgresql': {'data_dir': '.', 'parameters': {}, 'retry_timeout': 5},
'restapi': {'auth': 'u:p', 'listen': ''}, 'etcd': {'host': 'localhost:2379'}}))
class TestCtl(unittest.TestCase):
@@ -66,92 +64,76 @@ class TestCtl(unittest.TestCase):
self.assertRaises(PatroniCtlException, parse_dcs, 'invalid://test')
def test_output_members(self):
scheduled_at = datetime.now(tzutc) + timedelta(seconds=600)
cluster = get_cluster_initialized_with_leader(Failover(1, 'foo', 'bar', scheduled_at))
cluster = get_cluster_initialized_with_leader()
self.assertIsNone(output_members(cluster, name='abc', fmt='pretty'))
self.assertIsNone(output_members(cluster, name='abc', fmt='json'))
self.assertIsNone(output_members(cluster, name='abc', fmt='yaml'))
self.assertIsNone(output_members(cluster, name='abc', fmt='tsv'))
@patch('patroni.ctl.get_dcs')
@patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
def test_switchover(self, mock_get_dcs):
mock_get_dcs.return_value = self.e
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
mock_get_dcs.return_value.set_failover_value = Mock()
result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
assert 'leader' in result.output
result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
assert result.exit_code == 0
with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00'])
assert result.exit_code == 1
# Aborting switchover, as we anser NO to the confirmation
result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\nN')
assert result.exit_code == 1
# Target and source are equal
result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nleader\n\ny')
assert result.exit_code == 1
# Reality is not part of this cluster
result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nReality\n\ny')
assert result.exit_code == 1
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force'])
assert 'Member' in result.output
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
assert result.exit_code == 0
# Invalid timestamp
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', 'invalid'])
assert result.exit_code != 0
# Invalid timestamp
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', '2115-02-30T12:00:00+01:00'])
assert result.exit_code != 0
# Specifying wrong leader
result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='dummy')
assert result.exit_code == 1
with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)):
# Non-responding patroni
result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
assert 'falling back to DCS' in result.output
with patch('patroni.ctl.request_patroni') as mocked:
mocked.return_value.status_code = 500
result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
assert 'Switchover failed' in result.output
mocked.return_value.status_code = 501
mocked.return_value.text = 'Server does not support this operation'
result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
assert 'Switchover failed' in result.output
# No members available
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
assert result.exit_code == 1
# No master available
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
assert result.exit_code == 1
@patch('patroni.ctl.get_dcs')
@patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
def test_failover(self, mock_get_dcs):
mock_get_dcs.return_value = self.e
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
mock_get_dcs.return_value.set_failover_value = Mock()
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='\n')
assert 'Failover could be performed only to a specific candidate' in result.output
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
assert 'leader' in result.output
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
assert result.exit_code == 0
with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00'])
assert result.exit_code == 1
# Aborting failover,as we anser NO to the confirmation
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\nN')
assert result.exit_code == 1
# Target and source are equal
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nleader\n\ny')
assert result.exit_code == 1
# Reality is not part of this cluster
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nReality\n\ny')
assert result.exit_code == 1
result = self.runner.invoke(ctl, ['failover', 'dummy', '--force'])
assert 'Member' in result.output
result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
assert result.exit_code == 0
# Invalid timestamp
result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', 'invalid'])
assert result.exit_code != 0
# Invalid timestamp
result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2115-02-30T12:00:00+01:00'])
assert result.exit_code != 0
# Specifying wrong leader
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='dummy')
assert result.exit_code == 1
with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)):
# Non-responding patroni
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
assert 'falling back to DCS' in result.output
with patch('patroni.ctl.request_patroni') as mocked:
mocked.return_value.status_code = 500
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
assert 'Failover failed' in result.output
# No members available
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
assert result.exit_code == 1
# No master available
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
assert result.exit_code == 1
def test_get_dcs(self):
self.assertRaises(PatroniCtlException, get_dcs, {'dummy': {}}, 'dummy')
@@ -192,24 +174,24 @@ class TestCtl(unittest.TestCase):
def test_query_member(self):
with patch('patroni.ctl.get_cursor', Mock(return_value=MockConnect().cursor())):
rows = query_member(None, None, None, 'master', 'SELECT pg_catalog.pg_is_in_recovery()', {})
rows = query_member(None, None, None, 'master', 'SELECT pg_is_in_recovery()', {})
self.assertTrue('False' in str(rows))
rows = query_member(None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
self.assertEqual(rows, (None, None))
rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()', {})
self.assertEquals(rows, (None, None))
with patch('test_postgresql.MockCursor.execute', Mock(side_effect=OperationalError('bla'))):
rows = query_member(None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()', {})
with patch('patroni.ctl.get_cursor', Mock(return_value=None)):
rows = query_member(None, None, None, None, 'SELECT pg_catalog.pg_is_in_recovery()', {})
rows = query_member(None, None, None, None, 'SELECT pg_is_in_recovery()', {})
self.assertTrue('No connection to' in str(rows))
rows = query_member(None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()', {})
self.assertTrue('No connection to' in str(rows))
with patch('patroni.ctl.get_cursor', Mock(side_effect=OperationalError('bla'))):
rows = query_member(None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()', {})
@patch('patroni.ctl.get_dcs')
def test_dsn(self, mock_get_dcs):
@@ -225,22 +207,6 @@ class TestCtl(unittest.TestCase):
result = self.runner.invoke(ctl, ['dsn', 'alpha', '--member', 'dummy'])
assert result.exit_code == 1
@patch('requests.post')
@patch('patroni.ctl.get_dcs')
def test_reload(self, mock_get_dcs, mock_post):
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
result = self.runner.invoke(ctl, ['reload', 'alpha'], input='y')
assert 'Failed: reload for member' in result.output
mock_post.return_value.status_code = 200
result = self.runner.invoke(ctl, ['reload', 'alpha'], input='y')
assert 'No changes to apply on member' in result.output
mock_post.return_value.status_code = 202
result = self.runner.invoke(ctl, ['reload', 'alpha'], input='y')
assert 'Reload request received for member' in result.output
@patch('requests.post', requests_get)
@patch('patroni.ctl.get_dcs')
def test_restart_reinit(self, mock_get_dcs):
@@ -253,7 +219,7 @@ class TestCtl(unittest.TestCase):
assert result.exit_code == 1
# successful reinit
result = self.runner.invoke(ctl, ['reinit', 'alpha', 'other'], input='y\ny')
result = self.runner.invoke(ctl, ['reinit', 'alpha', 'other'], input='y')
assert result.exit_code == 0
# Aborted restart
@@ -269,7 +235,7 @@ class TestCtl(unittest.TestCase):
# Wrong pg version
result = self.runner.invoke(ctl, ['restart', 'alpha', '--any', '--pg-version', '9.1'], input='y')
assert 'Error: Invalid PostgreSQL version format' in result.output
assert 'Error: PostgreSQL version' in result.output
assert result.exit_code == 1
result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force', '--timeout', '10min'])
@@ -295,7 +261,7 @@ class TestCtl(unittest.TestCase):
with patch('requests.post', Mock(return_value=MockResponse(204))):
# get restart with the non-200 return code
# normal restart, the schedule is actually parsed, but not validated in patronictl
result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0',
result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0.0',
'--scheduled', '2300-10-01T14:30'], input='y')
assert result.exit_code == 0
@@ -344,14 +310,8 @@ class TestCtl(unittest.TestCase):
assert result.exit_code == 0
@patch('requests.post', Mock(side_effect=requests.exceptions.ConnectionError('foo')))
@patch('click.get_current_context')
def test_request_patroni(self, mock_context):
def test_request_patroni(self):
member = get_cluster_initialized_with_leader().leader.member
mock_context.return_value.obj = {'ctl': {'cacert': 'cert.pem'}}
self.assertRaises(requests.exceptions.ConnectionError, request_patroni, member, 'post', 'dummy', {})
mock_context.return_value.obj = {'ctl': {'insecure': True}}
self.assertRaises(requests.exceptions.ConnectionError, request_patroni, member, 'post', 'dummy', {})
def test_ctl(self):
@@ -364,29 +324,27 @@ class TestCtl(unittest.TestCase):
self.assertIsNone(get_any_member(get_cluster_initialized_without_leader(), role='master'))
m = get_any_member(get_cluster_initialized_with_leader(), role='master')
self.assertEqual(m.name, 'leader')
self.assertEquals(m.name, 'leader')
def test_get_all_members(self):
self.assertEqual(list(get_all_members(get_cluster_initialized_without_leader(), role='master')), [])
self.assertEquals(list(get_all_members(get_cluster_initialized_without_leader(), role='master')), [])
r = list(get_all_members(get_cluster_initialized_with_leader(), role='master'))
self.assertEqual(len(r), 1)
self.assertEqual(r[0].name, 'leader')
self.assertEquals(len(r), 1)
self.assertEquals(r[0].name, 'leader')
r = list(get_all_members(get_cluster_initialized_with_leader(), role='replica'))
self.assertEqual(len(r), 1)
self.assertEqual(r[0].name, 'other')
self.assertEquals(len(r), 1)
self.assertEquals(r[0].name, 'other')
self.assertEqual(len(list(get_all_members(get_cluster_initialized_without_leader(), role='replica'))), 2)
self.assertEquals(len(list(get_all_members(get_cluster_initialized_without_leader(), role='replica'))), 2)
@patch('patroni.ctl.get_dcs')
def test_members(self, mock_get_dcs):
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
result = self.runner.invoke(ctl, ['list'])
result = self.runner.invoke(members, ['alpha'])
assert '127.0.0.1' in result.output
assert result.exit_code == 0
with patch('patroni.ctl.load_config', Mock(return_value={})):
self.runner.invoke(ctl, ['list'])
def test_configure(self):
result = self.runner.invoke(configure, ['--dcs', 'abc', '-c', 'dummy', '-n', 'bla'])
@@ -422,7 +380,7 @@ class TestCtl(unittest.TestCase):
cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
result = self.runner.invoke(ctl, ['list', 'dummy', '--extended', '--timestamp'])
result = self.runner.invoke(ctl, ['list', 'dummy', '--extended'])
assert '2100' in result.output
assert 'Scheduled restart' in result.output
@@ -442,11 +400,14 @@ class TestCtl(unittest.TestCase):
assert 'Failed: flush scheduled restart' in result.output
@patch('patroni.ctl.get_dcs')
@patch('patroni.ctl.polling_loop', Mock(return_value=[1]))
def test_pause_cluster(self, mock_get_dcs):
mock_get_dcs.return_value = self.e
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
with patch('requests.patch', Mock(return_value=MockResponse(200))):
result = self.runner.invoke(ctl, ['pause', 'dummy'])
assert 'Success' in result.output
with patch('requests.patch', Mock(return_value=MockResponse(500))):
result = self.runner.invoke(ctl, ['pause', 'dummy'])
assert 'Failed' in result.output
@@ -456,17 +417,6 @@ class TestCtl(unittest.TestCase):
result = self.runner.invoke(ctl, ['pause', 'dummy'])
assert 'Cluster is already paused' in result.output
with patch('requests.patch', Mock(return_value=MockResponse(200))):
result = self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
assert "'pause' request sent" in result.output
mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
get_cluster(None, None, [], None, None)])
self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
member = Member(1, 'other', 28, {})
mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
get_cluster(None, None, [member], None, None)])
self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
@patch('patroni.ctl.get_dcs')
def test_resume_cluster(self, mock_get_dcs):
mock_get_dcs.return_value = self.e
@@ -499,23 +449,23 @@ class TestCtl(unittest.TestCase):
after_editing, changed_config = apply_config_changes(before_editing, config,
["postgresql.parameters.work_mem = 5MB",
"ttl=15", "postgresql.use_pg_rewind=off", 'a.b=c'])
self.assertEqual(changed_config, {"a": {"b": "c"}, "postgresql": {"parameters": {"work_mem": "5MB"},
"use_pg_rewind": False}, "ttl": 15})
self.assertEquals(changed_config, {"a": {"b": "c"}, "postgresql": {"parameters": {"work_mem": "5MB"},
"use_pg_rewind": False}, "ttl": 15})
# postgresql.parameters namespace is flattened
after_editing, changed_config = apply_config_changes(before_editing, config,
["postgresql.parameters.work_mem.sub = x"])
self.assertEqual(changed_config, {"postgresql": {"parameters": {"work_mem": "4MB", "work_mem.sub": "x"},
"use_pg_rewind": True}, "ttl": 30})
self.assertEquals(changed_config, {"postgresql": {"parameters": {"work_mem": "4MB", "work_mem.sub": "x"},
"use_pg_rewind": True}, "ttl": 30})
# Setting to null deletes
after_editing, changed_config = apply_config_changes(before_editing, config,
["postgresql.parameters.work_mem=null"])
self.assertEqual(changed_config, {"postgresql": {"use_pg_rewind": True}, "ttl": 30})
self.assertEquals(changed_config, {"postgresql": {"use_pg_rewind": True}, "ttl": 30})
after_editing, changed_config = apply_config_changes(before_editing, config,
["postgresql.use_pg_rewind=null",
"postgresql.parameters.work_mem=null"])
self.assertEqual(changed_config, {"ttl": 30})
self.assertEquals(changed_config, {"ttl": 30})
self.assertRaises(PatroniCtlException, apply_config_changes, before_editing, config, ['a'])
@@ -556,21 +506,3 @@ class TestCtl(unittest.TestCase):
self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar')
mock_get_dcs.return_value.set_config_value = Mock(return_value=True)
self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar')
@patch('patroni.ctl.get_dcs')
def test_version(self, mock_get_dcs):
mock_get_dcs.return_value = self.e
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
with patch('patroni.ctl.request_patroni') as mocked:
result = self.runner.invoke(ctl, ['version'])
assert 'patronictl version' in result.output
mocked.return_value.json = lambda: {'patroni': {'version': '1.2.3'}, 'server_version': 100001}
result = self.runner.invoke(ctl, ['version', 'dummy'])
assert '1.2.3' in result.output
with patch('requests.get', Mock(side_effect=Exception)):
result = self.runner.invoke(ctl, ['version', 'dummy'])
assert 'failed to get version' in result.output
def test_format_pg_version(self):
self.assertEqual(format_pg_version(100001), '10.1')
self.assertEqual(format_pg_version(90605), '9.6.5')
+7 -24
View File
@@ -18,6 +18,7 @@ class MockResponse(object):
self.status_code = status_code
self.content = '{}'
self.ok = True
self.text = ''
def json(self):
return json.loads(self.content)
@@ -26,10 +27,6 @@ class MockResponse(object):
def data(self):
return self.content.encode('utf-8')
@property
def text(self):
return self.content
@property
def status(self):
return self.status_code
@@ -51,12 +48,6 @@ def requests_get(url, **kwargs):
response.content = '[{}]' if url.startswith('http://error') else members
elif url.startswith('http://exhibitor'):
response.content = '{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}'
elif url.endswith(':8011/reinitialize'):
data = kwargs.get('data', '')
if ' false}' in data:
response.status_code = 503
response.ok = False
response.content = 'restarting after failure already in progress'
else:
response.status_code = 404
response.ok = False
@@ -90,7 +81,7 @@ def etcd_read(self, key, **kwargs):
raise etcd.EtcdKeyNotFound
response = {"action": "get", "node": {"key": "/service/batman5", "dir": True, "nodes": [
{"key": "/service/batman5/config", "value": '{"synchronous_mode": 0}',
{"key": "/service/batman5/config", "value": '{"foo": "bar"}',
"modifiedIndex": 1582, "createdIndex": 1582},
{"key": "/service/batman5/failover", "value": "",
"modifiedIndex": 1582, "createdIndex": 1582},
@@ -207,7 +198,6 @@ class TestClient(unittest.TestCase):
mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
self.client._machines_cache_updated = 0
self.client.api_execute('/', 'POST', timeout=0)
self.client._machines_cache = [self.client._base_uri]
self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
self.client._update_machines_cache = True
@@ -215,8 +205,8 @@ class TestClient(unittest.TestCase):
self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'GET')
def test_get_srv_record(self):
self.assertEqual(self.client.get_srv_record('_etcd-server._tcp.blabla'), [])
self.assertEqual(self.client.get_srv_record('_etcd-server._tcp.exception'), [])
self.assertEquals(self.client.get_srv_record('_etcd-server._tcp.blabla'), [])
self.assertEquals(self.client.get_srv_record('_etcd-server._tcp.exception'), [])
def test__get_machines_cache_from_srv(self):
self.client._get_machines_cache_from_srv('foobar')
@@ -259,7 +249,7 @@ class TestEtcd(unittest.TestCase):
'host': 'localhost:2379', 'scope': 'test', 'name': 'foo'})
def test_base_path(self):
self.assertEqual(self.etcd._base_path, '/patroni/test')
self.assertEquals(self.etcd._base_path, '/patroni/test')
@patch('dns.resolver.query', dns_query)
def test_get_etcd_client(self):
@@ -272,13 +262,9 @@ class TestEtcd(unittest.TestCase):
{'url': 'https://test:2379', 'retry_timeout': 10})
self.assertRaises(SleepException, self.etcd.get_etcd_client,
{'proxy': 'https://user:password@test:2379', 'retry_timeout': 10})
self.assertRaises(SleepException, self.etcd.get_etcd_client,
{'hosts': 'foo:4001,bar', 'retry_timeout': 10})
def test_get_cluster(self):
cluster = self.etcd.get_cluster()
self.assertIsInstance(cluster, Cluster)
self.assertFalse(cluster.is_synchronous_mode())
self.assertIsInstance(self.etcd.get_cluster(), Cluster)
self.etcd._base_path = '/service/nocluster'
cluster = self.etcd.get_cluster()
self.assertIsInstance(cluster, Cluster)
@@ -302,7 +288,7 @@ class TestEtcd(unittest.TestCase):
self.etcd.write_leader_optime('0')
def test_update_leader(self):
self.assertTrue(self.etcd.update_leader(None))
self.assertTrue(self.etcd.update_leader())
def test_initialize(self):
self.assertFalse(self.etcd.initialize())
@@ -338,6 +324,3 @@ class TestEtcd(unittest.TestCase):
def test_sync_state(self):
self.assertFalse(self.etcd.write_sync_state('leader', None))
self.assertFalse(self.etcd.delete_sync_state())
def test_set_history_value(self):
self.assertFalse(self.etcd.set_history_value('{}'))
+128 -311
View File
@@ -2,11 +2,10 @@ import datetime
import etcd
import os
import unittest
import sys
from mock import Mock, MagicMock, PropertyMock, patch
from patroni.config import Config
from patroni.dcs import Cluster, ClusterConfig, Failover, Leader, Member, get_dcs, SyncState, TimelineHistory
from patroni.dcs import Cluster, ClusterConfig, Failover, Leader, Member, get_dcs, SyncState
from patroni.dcs.etcd import Client
from patroni.exceptions import DCSError, PostgresConnectionException, PatroniException
from patroni.ha import Ha, _MemberStatus
@@ -14,9 +13,7 @@ from patroni.postgresql import Postgresql
from patroni.watchdog import Watchdog
from patroni.utils import tzutc
from test_etcd import socket_getaddrinfo, etcd_read, etcd_write, requests_get
from test_postgresql import psycopg2_connect, MockPostmaster
SYSID = '12345678901'
from test_postgresql import psycopg2_connect
def true(*args, **kwargs):
@@ -27,63 +24,45 @@ def false(*args, **kwargs):
return False
def get_cluster(initialize, leader, members, failover, sync, cluster_config=None):
t = datetime.datetime.now().isoformat()
history = TimelineHistory(1, '[[1,67197376,"no recovery target specified","' + t + '"]]',
[(1, 67197376, 'no recovery target specified', t)])
cluster_config = cluster_config or ClusterConfig(1, {'check_timeline': True}, 1)
return Cluster(initialize, cluster_config, leader, 10, members, failover, sync, history)
def get_cluster(initialize, leader, members, failover, sync):
return Cluster(initialize, ClusterConfig(1, {1: 2}, 1), leader, 10, members, failover, sync)
def get_cluster_not_initialized_without_leader(cluster_config=None):
return get_cluster(None, None, [], None, SyncState(None, None, None), cluster_config)
def get_cluster_not_initialized_without_leader():
return get_cluster(None, None, [], None, SyncState(None, None, None))
def get_cluster_initialized_without_leader(leader=False, failover=None, sync=None, cluster_config=None):
def get_cluster_initialized_without_leader(leader=False, failover=None, sync=None):
m1 = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres',
'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location': 4})
leader = Leader(0, 0, m1) if leader else None
l = Leader(0, 0, m1) if leader else None
m2 = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
'api_url': 'http://127.0.0.1:8011/patroni',
'state': 'running',
'pause': True,
'tags': {'clonefrom': True},
'scheduled_restart': {'schedule': "2100-01-01 10:53:07.560445+00:00",
'postgres_version': '99.0.0'}})
syncstate = SyncState(0 if sync else None, sync and sync[0], sync and sync[1])
return get_cluster(SYSID, leader, [m1, m2], failover, syncstate, cluster_config)
return get_cluster(True, l, [m1, m2], failover, syncstate)
def get_cluster_initialized_with_leader(failover=None, sync=None):
return get_cluster_initialized_without_leader(leader=True, failover=failover, sync=sync)
def get_cluster_initialized_with_only_leader(failover=None, cluster_config=None):
leader = get_cluster_initialized_without_leader(leader=True, failover=failover).leader
return get_cluster(True, leader, [leader], failover, None, cluster_config)
def get_cluster_initialized_with_only_leader(failover=None):
l = get_cluster_initialized_without_leader(leader=True, failover=failover).leader
return get_cluster(True, l, [l], failover, None)
def get_standby_cluster_initialized_with_only_leader(failover=None, sync=None):
return get_cluster_initialized_with_only_leader(
cluster_config=ClusterConfig(1, {
"standby_cluster": {
"host": "localhost",
"port": 5432,
"primary_slot_name": "",
}}, 1)
)
def get_node_status(reachable=True, in_recovery=True, timeline=2,
wal_position=10, nofailover=False, watchdog_failed=False):
def get_node_status(reachable=True, in_recovery=True, wal_position=10, nofailover=False, watchdog_failed=False):
def fetch_node_status(e):
tags = {}
if nofailover:
tags['nofailover'] = True
return _MemberStatus(e, reachable, in_recovery, timeline, wal_position, tags, watchdog_failed)
return _MemberStatus(e, reachable, in_recovery, wal_position, tags, watchdog_failed)
return fetch_node_status
future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
postmaster_start_time = datetime.datetime.now(tzutc)
@@ -113,12 +92,7 @@ zookeeper:
hosts: [localhost]
port: 8181
"""
# We rely on sys.argv in Config, so it's necessary to reset
# all the extra values that are coming from py.test
sys.argv = sys.argv[:1]
self.config = Config()
self.config.set_dynamic_configuration({'maximum_lag_on_failover': 5})
self.postgresql = p
self.dcs = d
self.api = Mock()
@@ -137,25 +111,22 @@ def run_async(self, func, args=()):
return func(*args) if args else func()
@patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
@patch.object(Postgresql, 'is_leader', Mock(return_value=True))
@patch.object(Postgresql, 'timeline_wal_position', Mock(return_value=(1, 10)))
@patch.object(Postgresql, '_cluster_info_state_get', Mock(return_value=3))
@patch.object(Postgresql, 'wal_position', Mock(return_value=10))
@patch.object(Postgresql, 'call_nowait', Mock(return_value=True))
@patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
@patch.object(Postgresql, 'controldata', Mock(return_value={'Database system identifier': SYSID}))
@patch.object(Postgresql, 'controldata', Mock(return_value={'Database system identifier': '1234567890'}))
@patch.object(Postgresql, 'sync_replication_slots', Mock())
@patch.object(Postgresql, 'write_pg_hba', Mock())
@patch.object(Postgresql, 'write_pgpass', Mock(return_value={}))
@patch.object(Postgresql, 'write_recovery_conf', Mock())
@patch.object(Postgresql, 'query', Mock())
@patch.object(Postgresql, 'checkpoint', Mock())
@patch.object(Postgresql, 'cancellable_subprocess_call', Mock(return_value=0))
@patch.object(Postgresql, '_get_local_timeline_lsn_from_replication_connection', Mock(return_value=[2, 10]))
@patch.object(Postgresql, 'call_nowait', Mock())
@patch.object(etcd.Client, 'write', etcd_write)
@patch.object(etcd.Client, 'read', etcd_read)
@patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
@patch('patroni.postgresql.polling_loop', Mock(return_value=range(1)))
@patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=False))
@patch('patroni.async_executor.AsyncExecutor.run_async', run_async)
@patch('subprocess.call', Mock(return_value=0))
@@ -164,13 +135,14 @@ class TestHa(unittest.TestCase):
@patch('socket.getaddrinfo', socket_getaddrinfo)
@patch('psycopg2.connect', psycopg2_connect)
@patch('patroni.dcs.dcs_modules', Mock(return_value=['patroni.dcs.foo', 'patroni.dcs.etcd']))
@patch('patroni.dcs.dcs_modules', Mock(return_value=['foo', 'patroni.dcs.etcd']))
@patch.object(etcd.Client, 'read', etcd_read)
def setUp(self):
with patch.object(Client, 'machines') as mock_machines:
mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
self.p = Postgresql({'name': 'postgresql0', 'scope': 'dummy', 'listen': '127.0.0.1:5432',
'data_dir': 'data/postgresql0', 'retry_timeout': 10,
'maximum_lag_on_failover': 5,
'authentication': {'superuser': {'username': 'foo', 'password': 'bar'},
'replication': {'username': '', 'password': ''}},
'parameters': {'wal_level': 'hot_standby', 'max_replication_slots': 5, 'foo': 'bar',
@@ -183,82 +155,47 @@ class TestHa(unittest.TestCase):
'name': 'foo', 'retry_timeout': 10}})
self.ha = Ha(MockPatroni(self.p, self.e))
self.ha.old_cluster = self.e.get_cluster()
self.ha.cluster = get_cluster_initialized_without_leader()
self.ha.cluster = get_cluster_not_initialized_without_leader()
self.ha.load_cluster_from_dcs = Mock()
self.ha.is_synchronous_mode = false
def test_update_lock(self):
self.p.last_operation = Mock(side_effect=PostgresConnectionException(''))
self.assertTrue(self.ha.update_lock(True))
def test_touch_member(self):
self.p.timeline_wal_position = Mock(return_value=(0, 1))
self.p.replica_cached_timeline = Mock(side_effect=Exception)
self.p.wal_position = Mock(side_effect=Exception)
self.ha.touch_member()
def test_is_leader(self):
self.assertFalse(self.ha.is_leader())
def test_start_as_replica(self):
self.p.is_healthy = false
self.assertEqual(self.ha.run_cycle(), 'starting as a secondary')
@patch('patroni.dcs.etcd.Etcd.initialize', return_value=True)
def test_start_as_standby_leader(self, initialize):
self.p.data_directory_empty = true
self.ha.cluster = get_cluster_not_initialized_without_leader(cluster_config=ClusterConfig(0, {}, 0))
self.ha.cluster.is_unlocked = true
self.ha.patroni.config._dynamic_configuration = {"standby_cluster": {"port": 5432}}
self.assertEqual(self.ha.run_cycle(), 'trying to bootstrap a new standby leader')
@patch.object(Cluster, 'get_clone_member',
Mock(return_value=Member(0, 'test', 1, {'api_url': 'http://127.0.0.1:8011/patroni'})))
@patch.object(Postgresql, 'create_replica', Mock(return_value=0))
def test_start_as_cascade_replica_in_standby_cluster(self):
self.p.data_directory_empty = true
self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
self.ha.cluster.is_unlocked = false
self.assertEqual(self.ha.run_cycle(), "trying to bootstrap from replica 'test'")
self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
def test_recover_replica_failed(self):
self.p.controldata = lambda: {'Database cluster state': 'in recovery', 'Database system identifier': SYSID}
self.p.controldata = lambda: {'Database cluster state': 'in production'}
self.p.is_running = false
self.p.follow = false
self.assertEqual(self.ha.run_cycle(), 'starting as a secondary')
self.assertEqual(self.ha.run_cycle(), 'failed to start postgres')
self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
self.assertEquals(self.ha.run_cycle(), 'failed to start postgres')
def test_recover_former_master(self):
def test_recover_master_failed(self):
self.p.follow = false
self.p.is_running = false
self.p.name = 'leader'
self.p.set_role('master')
self.p.controldata = lambda: {'Database cluster state': 'shut down', 'Database system identifier': SYSID}
self.p.controldata = lambda: {'Database cluster state': 'in production'}
self.ha.cluster = get_cluster_initialized_with_leader()
self.assertEqual(self.ha.run_cycle(), 'starting as readonly because i had the session lock')
self.assertEquals(self.ha.run_cycle(), 'starting as readonly because i had the session lock')
@patch.object(Postgresql, 'fix_cluster_state', Mock())
def test_crash_recovery(self):
self.p.is_running = false
self.p.controldata = lambda: {'Database cluster state': 'in production', 'Database system identifier': SYSID}
self.assertEqual(self.ha.run_cycle(), 'doing crash recovery in a single user mode')
@patch.object(Postgresql, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
@patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
@patch.object(Postgresql, 'rewind_needed_and_possible', Mock(return_value=True))
def test_recover_with_rewind(self):
self.p.is_running = false
self.ha.cluster = get_cluster_initialized_with_leader()
self.assertEqual(self.ha.run_cycle(), 'running pg_rewind from leader')
@patch.object(Postgresql, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
@patch.object(Postgresql, 'create_replica', Mock(return_value=1))
def test_recover_with_reinitialize(self):
self.p.is_running = false
self.ha.cluster = get_cluster_initialized_with_leader()
self.assertEqual(self.ha.run_cycle(), 'reinitializing due to diverged timelines')
self.assertEquals(self.ha.run_cycle(), 'running pg_rewind from leader')
@patch('sys.exit', return_value=1)
@patch('patroni.ha.Ha.sysid_valid', MagicMock(return_value=True))
def test_sysid_no_match(self, exit_mock):
self.p.controldata = lambda: {'Database cluster state': 'in recovery', 'Database system identifier': '123'}
self.ha.run_cycle()
exit_mock.assert_called_once_with(1)
@@ -267,131 +204,120 @@ class TestHa(unittest.TestCase):
self.p.is_leader = false
self.p.is_healthy = true
self.ha.has_lock = true
self.p.controldata = lambda: {'Database cluster state': 'in production', 'Database system identifier': SYSID}
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader because i had the session lock')
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader because i had the session lock')
@patch('psycopg2.connect', psycopg2_connect)
def test_acquire_lock_as_master(self):
self.assertEqual(self.ha.run_cycle(), 'acquired session lock as a leader')
self.assertEquals(self.ha.run_cycle(), 'acquired session lock as a leader')
def test_promoted_by_acquiring_lock(self):
self.ha.is_healthiest_node = true
self.p.is_leader = false
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
def test_long_promote(self):
self.ha.cluster.is_unlocked = false
self.ha.has_lock = true
self.p.is_leader = false
self.p.set_role('master')
self.assertEqual(self.ha.run_cycle(), 'no action. i am the leader with the lock')
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
def test_demote_after_failing_to_obtain_lock(self):
self.ha.acquire_lock = false
self.assertEqual(self.ha.run_cycle(), 'demoted self after trying and failing to obtain lock')
self.assertEquals(self.ha.run_cycle(), 'demoted self after trying and failing to obtain lock')
def test_follow_new_leader_after_failing_to_obtain_lock(self):
self.ha.is_healthiest_node = true
self.ha.acquire_lock = false
self.p.is_leader = false
self.assertEqual(self.ha.run_cycle(), 'following new leader after trying and failing to obtain lock')
self.assertEquals(self.ha.run_cycle(), 'following new leader after trying and failing to obtain lock')
def test_demote_because_not_healthiest(self):
self.ha.is_healthiest_node = false
self.assertEqual(self.ha.run_cycle(), 'demoting self because i am not the healthiest node')
self.assertEquals(self.ha.run_cycle(), 'demoting self because i am not the healthiest node')
def test_follow_new_leader_because_not_healthiest(self):
self.ha.is_healthiest_node = false
self.p.is_leader = false
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
def test_promote_because_have_lock(self):
self.ha.cluster.is_unlocked = false
self.ha.has_lock = true
self.p.is_leader = false
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader because i had the session lock')
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader because i had the session lock')
def test_promote_without_watchdog(self):
self.ha.cluster.is_unlocked = false
self.ha.has_lock = true
self.p.is_leader = true
with patch.object(Watchdog, 'activate', Mock(return_value=False)):
self.assertEqual(self.ha.run_cycle(), 'Demoting self because watchdog could not be activated')
self.assertEquals(self.ha.run_cycle(), 'Demoting self because watchdog could not be activated')
self.p.is_leader = false
self.assertEqual(self.ha.run_cycle(), 'Not promoting self because watchdog could not be activated')
self.assertEquals(self.ha.run_cycle(), 'Not promoting self because watchdog could not be actived')
def test_leader_with_lock(self):
self.ha.cluster = get_cluster_not_initialized_without_leader()
self.ha.cluster.is_unlocked = false
self.ha.has_lock = true
self.assertEqual(self.ha.run_cycle(), 'no action. i am the leader with the lock')
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
def test_demote_because_not_having_lock(self):
self.ha.cluster.is_unlocked = false
with patch.object(Watchdog, 'is_running', PropertyMock(return_value=True)):
self.assertEqual(self.ha.run_cycle(), 'demoting self because i do not have the lock and i was a leader')
self.assertEquals(self.ha.run_cycle(), 'demoting self because i do not have the lock and i was a leader')
def test_demote_because_update_lock_failed(self):
self.ha.cluster.is_unlocked = false
self.ha.has_lock = true
self.ha.update_lock = false
self.assertEqual(self.ha.run_cycle(), 'demoted self because failed to update leader lock in DCS')
self.assertEquals(self.ha.run_cycle(), 'demoted self because failed to update leader lock in DCS')
self.p.is_leader = false
self.assertEqual(self.ha.run_cycle(), 'not promoting because failed to update leader lock in DCS')
self.assertEquals(self.ha.run_cycle(), 'not promoting because failed to update leader lock in DCS')
def test_follow(self):
self.ha.cluster.is_unlocked = false
self.p.is_leader = false
self.assertEqual(self.ha.run_cycle(), 'no action. i am a secondary and i am following a leader')
self.assertEquals(self.ha.run_cycle(), 'no action. i am a secondary and i am following a leader')
self.ha.patroni.replicatefrom = "foo"
self.assertEqual(self.ha.run_cycle(), 'no action. i am a secondary and i am following a leader')
self.assertEquals(self.ha.run_cycle(), 'no action. i am a secondary and i am following a leader')
def test_follow_in_pause(self):
self.ha.cluster.is_unlocked = false
self.ha.is_paused = true
self.assertEqual(self.ha.run_cycle(), 'PAUSE: continue to run as master without lock')
self.assertEquals(self.ha.run_cycle(), 'PAUSE: continue to run as master without lock')
self.p.is_leader = false
self.assertEqual(self.ha.run_cycle(), 'PAUSE: no action')
self.assertEquals(self.ha.run_cycle(), 'PAUSE: no action')
@patch.object(Postgresql, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
@patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
@patch.object(Postgresql, 'rewind_needed_and_possible', Mock(return_value=True))
def test_follow_triggers_rewind(self):
self.p.is_leader = false
self.p.trigger_check_diverged_lsn()
self.ha.cluster = get_cluster_initialized_with_leader()
self.assertEqual(self.ha.run_cycle(), 'running pg_rewind from leader')
self.assertEquals(self.ha.run_cycle(), 'running pg_rewind from leader')
def test_no_etcd_connection_master_demote(self):
self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
self.assertEqual(self.ha.run_cycle(), 'demoted self because DCS is not accessible and i was a leader')
self.assertEquals(self.ha.run_cycle(), 'demoted self because DCS is not accessible and i was a leader')
@patch('time.sleep', Mock())
def test_bootstrap_from_another_member(self):
self.ha.cluster = get_cluster_initialized_with_leader()
self.assertEqual(self.ha.bootstrap(), 'trying to bootstrap from replica \'other\'')
self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap from replica \'other\'')
def test_bootstrap_waiting_for_leader(self):
self.ha.cluster = get_cluster_initialized_without_leader()
self.assertEqual(self.ha.bootstrap(), 'waiting for leader to bootstrap')
self.assertEquals(self.ha.bootstrap(), 'waiting for leader to bootstrap')
def test_bootstrap_without_leader(self):
self.ha.cluster = get_cluster_initialized_without_leader()
self.p.can_create_replica_without_replication_connection = MagicMock(return_value=True)
self.assertEqual(self.ha.bootstrap(), 'trying to bootstrap (without leader)')
self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap (without leader)')
def test_bootstrap_initialize_lock_failed(self):
self.ha.cluster = get_cluster_not_initialized_without_leader()
self.assertEqual(self.ha.bootstrap(), 'failed to acquire initialize lock')
self.assertEquals(self.ha.bootstrap(), 'failed to acquire initialize lock')
def test_bootstrap_initialized_new_cluster(self):
self.ha.cluster = get_cluster_not_initialized_without_leader()
self.e.initialize = true
self.assertEqual(self.ha.bootstrap(), 'trying to bootstrap a new cluster')
self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap a new cluster')
self.p.is_leader = false
self.assertEqual(self.ha.run_cycle(), 'waiting for end of recovery after bootstrap')
self.assertEquals(self.ha.run_cycle(), 'waiting for end of recovery after bootstrap')
self.p.is_leader = true
self.assertEqual(self.ha.run_cycle(), 'running post_bootstrap')
self.assertEqual(self.ha.run_cycle(), 'initialized a new cluster')
self.assertEquals(self.ha.run_cycle(), 'running post_bootstrap')
self.assertEquals(self.ha.run_cycle(), 'initialized a new cluster')
def test_bootstrap_release_initialize_key_on_failure(self):
self.ha.cluster = get_cluster_not_initialized_without_leader()
@@ -404,10 +330,10 @@ class TestHa(unittest.TestCase):
self.ha.cluster = get_cluster_not_initialized_without_leader()
self.e.initialize = true
self.ha.bootstrap()
self.p.is_running.return_value = MockPostmaster()
self.p.is_running = true
self.p.is_leader = true
with patch.object(Watchdog, 'activate', Mock(return_value=False)):
self.assertEqual(self.ha.post_bootstrap(), 'running post_bootstrap')
self.assertEquals(self.ha.post_bootstrap(), 'running post_bootstrap')
self.assertRaises(PatroniException, self.ha.post_bootstrap)
@patch('psycopg2.connect', psycopg2_connect)
@@ -415,7 +341,7 @@ class TestHa(unittest.TestCase):
self.assertIsNotNone(self.ha.reinitialize())
self.ha.cluster = get_cluster_initialized_with_leader()
self.assertIsNone(self.ha.reinitialize(True))
self.assertIsNone(self.ha.reinitialize())
self.assertIsNotNone(self.ha.reinitialize())
@@ -424,63 +350,61 @@ class TestHa(unittest.TestCase):
@patch('time.sleep', Mock())
def test_restart(self):
self.assertEqual(self.ha.restart({}), (True, 'restarted successfully'))
self.assertEquals(self.ha.restart({}), (True, 'restarted successfully'))
self.p.restart = Mock(return_value=None)
self.assertEqual(self.ha.restart({}), (False, 'postgres is still starting'))
self.assertEquals(self.ha.restart({}), (False, 'postgres is still starting'))
self.p.restart = false
self.assertEqual(self.ha.restart({}), (False, 'restart failed'))
self.assertEquals(self.ha.restart({}), (False, 'restart failed'))
self.ha.cluster = get_cluster_initialized_with_leader()
self.ha.reinitialize()
self.assertEqual(self.ha.restart({}), (False, 'reinitialize already in progress'))
self.assertEquals(self.ha.restart({}), (False, 'reinitialize already in progress'))
with patch.object(self.ha, "restart_matches", return_value=False):
self.assertEqual(self.ha.restart({'foo': 'bar'}), (False, "restart conditions are not satisfied"))
self.assertEquals(self.ha.restart({'foo': 'bar'}), (False, "restart conditions are not satisfied"))
@patch('os.kill', Mock())
def test_restart_in_progress(self):
with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)):
self.ha.restart({}, run_async=True)
self.assertTrue(self.ha.restart_scheduled())
self.assertEqual(self.ha.run_cycle(), 'restart in progress')
self.assertEquals(self.ha.run_cycle(), 'restart in progress')
self.ha.cluster = get_cluster_initialized_with_leader()
self.assertEqual(self.ha.run_cycle(), 'restart in progress')
self.assertEquals(self.ha.run_cycle(), 'restart in progress')
self.ha.has_lock = true
self.assertEqual(self.ha.run_cycle(), 'updated leader lock during restart')
self.assertEquals(self.ha.run_cycle(), 'updated leader lock during restart')
self.ha.update_lock = false
self.p.set_role('master')
with patch('patroni.async_executor.CriticalTask.cancel', Mock(return_value=False)):
with patch('patroni.postgresql.Postgresql.terminate_starting_postmaster') as mock_terminate:
self.assertEqual(self.ha.run_cycle(), 'lost leader lock during restart')
mock_terminate.assert_called()
with patch('patroni.postgresql.Postgresql.stop') as stop_mock:
self.assertEquals(self.ha.run_cycle(), 'lost leader lock during restart')
stop_mock.assert_called()
@patch('requests.get', requests_get)
def test_manual_failover_from_leader(self):
self.ha.fetch_node_status = get_node_status()
self.ha.has_lock = true
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None))
self.assertEqual(self.ha.run_cycle(), 'no action. i am the leader with the lock')
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None))
self.assertEqual(self.ha.run_cycle(), 'no action. i am the leader with the lock')
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla', None))
self.assertEqual(self.ha.run_cycle(), 'no action. i am the leader with the lock')
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
f = Failover(0, self.p.name, '', None)
self.ha.cluster = get_cluster_initialized_with_leader(f)
self.assertEqual(self.ha.run_cycle(), 'manual failover: demoting myself')
self.p.rewind_or_reinitialize_needed_and_possible = true
self.assertEqual(self.ha.run_cycle(), 'manual failover: demoting myself')
self.assertEquals(self.ha.run_cycle(), 'manual failover: demoting myself')
self.p.rewind_needed_and_possible = true
self.assertEquals(self.ha.run_cycle(), 'manual failover: demoting myself')
self.ha.fetch_node_status = get_node_status(nofailover=True)
self.assertEqual(self.ha.run_cycle(), 'no action. i am the leader with the lock')
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
self.ha.fetch_node_status = get_node_status(watchdog_failed=True)
self.assertEqual(self.ha.run_cycle(), 'no action. i am the leader with the lock')
self.ha.fetch_node_status = get_node_status(timeline=1)
self.assertEqual(self.ha.run_cycle(), 'no action. i am the leader with the lock')
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
self.ha.fetch_node_status = get_node_status(wal_position=1)
self.assertEqual(self.ha.run_cycle(), 'no action. i am the leader with the lock')
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
# manual failover from the previous leader to us won't happen if we hold the nofailover flag
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, None))
self.assertEqual(self.ha.run_cycle(), 'no action. i am the leader with the lock')
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
# Failover scheduled time must include timezone
scheduled = datetime.datetime.now()
@@ -489,19 +413,19 @@ class TestHa(unittest.TestCase):
scheduled = datetime.datetime.utcnow().replace(tzinfo=tzutc)
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
self.assertEqual('no action. i am the leader with the lock', self.ha.run_cycle())
self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle())
scheduled = scheduled + datetime.timedelta(seconds=30)
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
self.assertEqual('no action. i am the leader with the lock', self.ha.run_cycle())
self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle())
scheduled = scheduled + datetime.timedelta(seconds=-600)
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
self.assertEqual('no action. i am the leader with the lock', self.ha.run_cycle())
self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle())
scheduled = None
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
self.assertEqual('no action. i am the leader with the lock', self.ha.run_cycle())
self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle())
@patch('requests.get', requests_get)
def test_manual_failover_from_leader_in_pause(self):
@@ -509,60 +433,47 @@ class TestHa(unittest.TestCase):
self.ha.is_paused = true
scheduled = datetime.datetime.now()
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
self.assertEqual('PAUSE: no action. i am the leader with the lock', self.ha.run_cycle())
self.assertEquals('PAUSE: no action. i am the leader with the lock', self.ha.run_cycle())
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, '', None))
self.assertEqual('PAUSE: no action. i am the leader with the lock', self.ha.run_cycle())
@patch('requests.get', requests_get)
def test_manual_failover_from_leader_in_synchronous_mode(self):
self.p.is_leader = true
self.ha.has_lock = true
self.ha.is_synchronous_mode = true
self.ha.is_failover_possible = false
self.ha.process_sync_replication = Mock()
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None), (self.p.name, None))
self.assertEqual('no action. i am the leader with the lock', self.ha.run_cycle())
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None), (self.p.name, 'a'))
self.ha.is_failover_possible = true
self.assertEqual('manual failover: demoting myself', self.ha.run_cycle())
self.assertEquals('PAUSE: no action. i am the leader with the lock', self.ha.run_cycle())
@patch('requests.get', requests_get)
def test_manual_failover_process_no_leader(self):
self.p.is_leader = false
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', self.p.name, None))
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None))
self.p.set_role('replica')
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
self.ha.fetch_node_status = get_node_status() # accessible, in_recovery
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, self.p.name, '', None))
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
self.ha.fetch_node_status = get_node_status(reachable=False) # inaccessible, in_recovery
self.p.set_role('replica')
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
# set failover flag to True for all members of the cluster
# this should elect the current member, as we are not going to call the API for it.
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
self.ha.fetch_node_status = get_node_status(nofailover=True) # accessible, in_recovery
self.p.set_role('replica')
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
# same as previous, but set the current member to nofailover. In no case it should be elected as a leader
self.ha.patroni.nofailover = True
self.assertEqual(self.ha.run_cycle(), 'following a different leader because I am not allowed to promote')
self.assertEquals(self.ha.run_cycle(), 'following a different leader because I am not allowed to promote')
def test_manual_failover_process_no_leader_in_pause(self):
self.ha.is_paused = true
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
self.assertEqual(self.ha.run_cycle(), 'PAUSE: continue to run as master without lock')
self.assertEquals(self.ha.run_cycle(), 'PAUSE: continue to run as master without lock')
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', '', None))
self.assertEqual(self.ha.run_cycle(), 'PAUSE: continue to run as master without lock')
self.assertEquals(self.ha.run_cycle(), 'PAUSE: continue to run as master without lock')
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'blabla', None))
self.assertEqual('PAUSE: acquired session lock as a leader', self.ha.run_cycle())
self.assertEquals('PAUSE: acquired session lock as a leader', self.ha.run_cycle())
self.p.is_leader = false
self.p.set_role('replica')
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', self.p.name, None))
self.assertEqual(self.ha.run_cycle(), 'PAUSE: promoted self to leader by acquiring session lock')
self.assertEquals(self.ha.run_cycle(), 'PAUSE: promoted self to leader by acquiring session lock')
def test_is_healthiest_node(self):
self.ha.state_handler.is_leader = false
@@ -585,9 +496,7 @@ class TestHa(unittest.TestCase):
self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
self.ha.fetch_node_status = get_node_status(wal_position=11) # accessible, in_recovery, wal position ahead
self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
with patch('patroni.postgresql.Postgresql.timeline_wal_position', return_value=(1, 1)):
self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
with patch('patroni.postgresql.Postgresql.replica_cached_timeline', return_value=1):
with patch('patroni.postgresql.Postgresql.wal_position', return_value=1):
self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
self.ha.patroni.nofailover = True
self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
@@ -645,7 +554,7 @@ class TestHa(unittest.TestCase):
def test_scheduled_restart(self):
self.ha.cluster = get_cluster_initialized_with_leader()
with patch.object(self.ha, "evaluate_scheduled_restart", Mock(return_value="restart scheduled")):
self.assertEqual(self.ha.run_cycle(), "restart scheduled")
self.assertEquals(self.ha.run_cycle(), "restart scheduled")
def test_restart_matches(self):
self.p._role = 'replica'
@@ -662,80 +571,21 @@ class TestHa(unittest.TestCase):
self.ha.is_paused = true
self.p.name = 'leader'
self.ha.cluster = get_cluster_initialized_with_leader()
self.assertEqual(self.ha.run_cycle(), 'PAUSE: removed leader lock because postgres is not running as master')
self.assertEquals(self.ha.run_cycle(), 'PAUSE: removed leader lock because postgres is not running as master')
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None))
self.assertEqual(self.ha.run_cycle(), 'PAUSE: waiting to become master after promote...')
def test_process_healthy_standby_cluster_as_standby_leader(self):
self.p.is_leader = false
self.p.name = 'leader'
self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
msg = 'no action. i am the standby leader with the lock'
self.assertEqual(self.ha.run_cycle(), msg)
def test_process_healthy_standby_cluster_as_cascade_replica(self):
self.p.is_leader = false
self.p.name = 'replica'
self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
msg = 'no action. i am a secondary and i am following a leader'
self.assertEqual(self.ha.run_cycle(), msg)
def test_process_unhealthy_standby_cluster_as_standby_leader(self):
self.p.is_leader = false
self.p.name = 'leader'
self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
self.ha.cluster.is_unlocked = true
self.ha.sysid_valid = true
self.p._sysid = True
msg = 'promoted self to a standby leader because i had the session lock'
self.assertEqual(self.ha.run_cycle(), msg)
@patch.object(Postgresql, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
@patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
def test_process_unhealthy_standby_cluster_as_cascade_replica(self):
self.p.is_leader = false
self.p.name = 'replica'
self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
self.ha.is_unlocked = true
self.assertTrue(self.ha.run_cycle().startswith('running pg_rewind from remote_master:'))
def test_recover_unhealthy_leader_in_standby_cluster(self):
self.p.is_leader = false
self.p.name = 'leader'
self.p.is_running = false
self.p.follow = false
self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
self.assertEqual(self.ha.run_cycle(), 'starting as a standby leader because i had the session lock')
def test_recover_unhealthy_unlocked_standby_cluster(self):
self.p.is_leader = false
self.p.name = 'leader'
self.p.is_running = false
self.p.follow = false
self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
self.ha.cluster.is_unlocked = true
self.ha.has_lock = false
self.assertEqual(self.ha.run_cycle(), 'trying to follow a remote master because standby cluster is unhealthy')
def test_failed_to_update_lock_in_pause(self):
self.ha.update_lock = false
self.ha.is_paused = true
self.p.name = 'leader'
self.ha.cluster = get_cluster_initialized_with_leader()
self.assertEqual(self.ha.run_cycle(),
'PAUSE: continue to run as master after failing to update leader lock in DCS')
self.assertEquals(self.ha.run_cycle(), 'PAUSE: waiting to become master after promote...')
def test_postgres_unhealthy_in_pause(self):
self.ha.is_paused = true
self.p.is_healthy = false
self.assertEqual(self.ha.run_cycle(), 'PAUSE: postgres is not running')
self.assertEquals(self.ha.run_cycle(), 'PAUSE: postgres is not running')
self.ha.has_lock = true
self.assertEqual(self.ha.run_cycle(), 'PAUSE: removed leader lock because postgres is not running')
self.assertEquals(self.ha.run_cycle(), 'PAUSE: removed leader lock because postgres is not running')
def test_no_etcd_connection_in_pause(self):
self.ha.is_paused = true
self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
self.assertEqual(self.ha.run_cycle(), 'PAUSE: DCS is not accessible')
self.assertEquals(self.ha.run_cycle(), 'PAUSE: DCS is not accessible')
@patch('patroni.ha.Ha.update_lock', return_value=True)
@patch('patroni.ha.Ha.demote')
@@ -751,26 +601,26 @@ class TestHa(unittest.TestCase):
self.ha.cluster = get_cluster_initialized_with_leader()
self.p.check_for_startup = true
self.p.time_in_state = lambda: 30
self.assertEqual(self.ha.run_cycle(), 'PostgreSQL is still starting up, 270 seconds until timeout')
self.assertEquals(self.ha.run_cycle(), 'PostgreSQL is still starting up, 270 seconds until timeout')
check_calls([(update_lock, True), (demote, False)])
self.p.time_in_state = lambda: 350
self.ha.fetch_node_status = get_node_status(reachable=False) # inaccessible, in_recovery
self.assertEqual(self.ha.run_cycle(),
'master start has timed out, but continuing to wait because failover is not possible')
self.assertEquals(self.ha.run_cycle(),
'master start has timed out, but continuing to wait because failover is not possible')
check_calls([(update_lock, True), (demote, False)])
self.ha.fetch_node_status = get_node_status() # accessible, in_recovery
self.assertEqual(self.ha.run_cycle(), 'stopped PostgreSQL because of startup timeout')
self.assertEquals(self.ha.run_cycle(), 'stopped PostgreSQL because of startup timeout')
check_calls([(update_lock, True), (demote, True)])
update_lock.return_value = False
self.assertEqual(self.ha.run_cycle(), 'stopped PostgreSQL while starting up because leader key was lost')
self.assertEquals(self.ha.run_cycle(), 'stopped PostgreSQL while starting up because leader key was lost')
check_calls([(update_lock, True), (demote, True)])
self.ha.has_lock = false
self.p.is_leader = false
self.assertEqual(self.ha.run_cycle(), 'no action. i am a secondary and i am following a leader')
self.assertEquals(self.ha.run_cycle(), 'no action. i am a secondary and i am following a leader')
check_calls([(update_lock, False), (demote, False)])
def test_manual_failover_while_starting(self):
@@ -779,18 +629,17 @@ class TestHa(unittest.TestCase):
f = Failover(0, self.p.name, '', None)
self.ha.cluster = get_cluster_initialized_with_leader(f)
self.ha.fetch_node_status = get_node_status() # accessible, in_recovery
self.assertEqual(self.ha.run_cycle(), 'manual failover: demoting myself')
self.assertEquals(self.ha.run_cycle(), 'manual failover: demoting myself')
@patch('patroni.ha.Ha.demote')
def test_failover_immediately_on_zero_master_start_timeout(self, demote):
self.p.is_running = false
self.ha.cluster = get_cluster_initialized_with_leader(sync=(self.p.name, 'other'))
self.ha.cluster.config.data['synchronous_mode'] = True
self.ha.cluster = get_cluster_initialized_with_leader()
self.ha.patroni.config.set_dynamic_configuration({'master_start_timeout': 0})
self.ha.has_lock = true
self.ha.update_lock = true
self.ha.fetch_node_status = get_node_status() # accessible, in_recovery
self.assertEqual(self.ha.run_cycle(), 'stopped PostgreSQL to fail over after a crash')
self.assertEquals(self.ha.run_cycle(), 'stopped PostgreSQL to fail over after a crash')
demote.assert_called_once()
@patch('patroni.postgresql.Postgresql.follow')
@@ -852,18 +701,18 @@ class TestHa(unittest.TestCase):
self.p.pick_synchronous_standby = Mock(return_value=('other2', True))
self.ha.run_cycle()
self.ha.dcs.get_cluster.assert_called_once()
self.assertEqual(self.ha.dcs.write_sync_state.call_count, 2)
self.assertEquals(self.ha.dcs.write_sync_state.call_count, 2)
# Test updating sync standby key failed due to race
self.ha.dcs.write_sync_state = Mock(side_effect=[True, False])
self.ha.run_cycle()
self.assertEqual(self.ha.dcs.write_sync_state.call_count, 2)
self.assertEquals(self.ha.dcs.write_sync_state.call_count, 2)
# Test changing sync standby failed due to race
self.ha.dcs.write_sync_state = Mock(return_value=True)
self.ha.dcs.get_cluster = Mock(return_value=get_cluster_initialized_with_leader(sync=('somebodyelse', None)))
self.ha.run_cycle()
self.assertEqual(self.ha.dcs.write_sync_state.call_count, 1)
self.assertEquals(self.ha.dcs.write_sync_state.call_count, 1)
# Test sync set to '*' when synchronous_mode_strict is enabled
mock_set_sync.reset_mock()
@@ -884,7 +733,7 @@ class TestHa(unittest.TestCase):
self.ha.cluster = get_cluster_initialized_with_leader(sync=('other', None))
# When we just became master nobody is sync
self.assertEqual(self.ha.enforce_master_role('msg', 'promote msg'), 'promote msg')
self.assertEquals(self.ha.enforce_master_role('msg', 'promote msg'), 'promote msg')
mock_set_sync.assert_called_once_with(None)
mock_write_sync.assert_called_once_with('leader', None, index=0)
@@ -912,7 +761,7 @@ class TestHa(unittest.TestCase):
self.ha.run_cycle()
mock_acquire.assert_not_called()
mock_follow.assert_called_once()
self.assertEqual(mock_follow.call_args[0][0], None)
self.assertEquals(mock_follow.call_args[0][0], None)
mock_write_sync.assert_not_called()
mock_follow.reset_mock()
@@ -963,15 +812,15 @@ class TestHa(unittest.TestCase):
def test_effective_tags(self):
self.ha._disable_sync = True
self.assertEqual(self.ha.get_effective_tags(), {'foo': 'bar', 'nosync': True})
self.assertEquals(self.ha.get_effective_tags(), {'foo': 'bar', 'nosync': True})
self.ha._disable_sync = False
self.assertEqual(self.ha.get_effective_tags(), {'foo': 'bar'})
self.assertEquals(self.ha.get_effective_tags(), {'foo': 'bar'})
def test_restore_cluster_config(self):
self.ha.cluster.config.data.clear()
self.ha.has_lock = true
self.ha.cluster.is_unlocked = false
self.assertEqual(self.ha.run_cycle(), 'no action. i am the leader with the lock')
self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock')
def test_watch(self):
self.ha.cluster = get_cluster_initialized_with_leader()
@@ -982,7 +831,6 @@ class TestHa(unittest.TestCase):
def test_shutdown(self):
self.p.is_running = false
self.ha.has_lock = true
self.ha.shutdown()
@patch('time.sleep', Mock())
@@ -990,40 +838,9 @@ class TestHa(unittest.TestCase):
self.ha.cluster = get_cluster_initialized_with_leader()
self.ha.has_lock = true
self.p.data_directory_empty = true
self.assertEqual(self.ha.run_cycle(), 'released leader key voluntarily as data dir empty and currently leader')
self.assertEqual(self.p.role, 'uninitialized')
self.assertEquals(self.ha.run_cycle(), 'released leader key voluntarily as data dir empty and currently leader')
# as has_lock is mocked out, we need to fake the leader key release
self.ha.has_lock = false
# will not say bootstrap from leader as replica can't self elect
self.assertEqual(self.ha.run_cycle(), "trying to bootstrap from replica 'other'")
def test_update_cluster_history(self):
self.p.get_master_timeline = Mock(return_value=1)
self.ha.has_lock = true
self.ha.cluster.is_unlocked = false
self.assertEqual(self.ha.run_cycle(), 'no action. i am the leader with the lock')
@patch('sys.exit', return_value=1)
def test_abort_join(self, exit_mock):
self.ha.cluster = get_cluster_not_initialized_without_leader()
self.p.is_leader = false
self.ha.run_cycle()
exit_mock.assert_called_once_with(1)
def test_after_pause(self):
self.ha.has_lock = true
self.ha.cluster.is_unlocked = false
self.ha.is_paused = true
self.assertEqual(self.ha.run_cycle(), 'PAUSE: no action. i am the leader with the lock')
self.ha.is_paused = false
self.assertEqual(self.ha.run_cycle(), 'no action. i am the leader with the lock')
@patch('psycopg2.connect', psycopg2_connect)
def test_permanent_logical_slots_after_promote(self):
config = ClusterConfig(1, {'slots': {'l': {'database': 'postgres', 'plugin': 'test_decoding'}}}, 1)
self.ha.cluster = get_cluster_initialized_without_leader(cluster_config=config)
self.assertEqual(self.ha.run_cycle(), 'acquired session lock as a leader')
self.ha.cluster = get_cluster_initialized_without_leader(leader=True, cluster_config=config)
self.ha.has_lock = true
self.assertEqual(self.ha.run_cycle(), 'no action. i am the leader with the lock')
self.assertEquals(self.ha.run_cycle(), "trying to bootstrap from replica 'other'")
-126
View File
@@ -1,126 +0,0 @@
import unittest
from mock import Mock, patch
from patroni.dcs.kubernetes import Kubernetes, KubernetesError, k8s_client, k8s_watch, RetryFailedError
def mock_list_namespaced_config_map(self, *args, **kwargs):
metadata = {'resource_version': '1', 'labels': {'f': 'b'}, 'name': 'test-config',
'annotations': {'initialize': '123', 'config': '{}'}}
items = [k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))]
metadata.update({'name': 'test-leader', 'annotations': {'optime': '1234', 'leader': 'p-0', 'ttl': '30s'}})
items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
metadata.update({'name': 'test-failover', 'annotations': {'leader': 'p-0'}})
items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
metadata.update({'name': 'test-sync', 'annotations': {'leader': 'p-0'}})
items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
metadata = k8s_client.V1ObjectMeta(resource_version='1')
return k8s_client.V1ConfigMapList(metadata=metadata, items=items)
def mock_list_namespaced_pod(self, *args, **kwargs):
metadata = k8s_client.V1ObjectMeta(resource_version='1', name='p-0', annotations={'status': '{}'})
items = [k8s_client.V1Pod(metadata=metadata)]
return k8s_client.V1PodList(items=items)
@patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', Mock())
@patch.object(k8s_client.CoreV1Api, 'create_namespaced_config_map', Mock())
class TestKubernetes(unittest.TestCase):
@patch('kubernetes.config.load_kube_config', Mock())
@patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map)
@patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', mock_list_namespaced_pod)
def setUp(self):
self.k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10, 'labels': {'f': 'b'}})
with patch('time.time', Mock(return_value=1)):
self.k.get_cluster()
@patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map)
@patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', mock_list_namespaced_pod)
def test_get_cluster(self):
self.k.get_cluster()
with patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', Mock(side_effect=Exception)):
self.assertRaises(KubernetesError, self.k.get_cluster)
@patch('kubernetes.config.load_kube_config', Mock())
@patch.object(k8s_client.CoreV1Api, 'create_namespaced_endpoints', Mock())
def test_update_leader(self):
k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10,
'labels': {'f': 'b'}, 'use_endpoints': True, 'pod_ip': '10.0.0.0'})
self.assertIsNotNone(k.update_leader('123'))
@patch('kubernetes.config.load_kube_config', Mock())
@patch.object(k8s_client.CoreV1Api, 'create_namespaced_endpoints', Mock())
def test_update_leader_with_restricted_access(self):
k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10,
'labels': {'f': 'b'}, 'use_endpoints': True, 'pod_ip': '10.0.0.0'})
self.assertIsNotNone(k.update_leader('123', True))
def test_take_leader(self):
self.k.take_leader()
self.k._leader_observed_record['leader'] = 'test'
self.k.patch_or_create = Mock(return_value=False)
self.k.take_leader()
def test_manual_failover(self):
with patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', Mock(side_effect=RetryFailedError(''))):
self.k.manual_failover('foo', 'bar')
def test_set_config_value(self):
self.k.set_config_value('{}')
@patch.object(k8s_client.CoreV1Api, 'patch_namespaced_pod', Mock(return_value=True))
def test_touch_member(self):
self.k.touch_member({'role': 'replica'})
self.k._name = 'p-1'
self.k.touch_member({'state': 'running', 'role': 'replica'})
self.k.touch_member({'state': 'stopped', 'role': 'master'})
def test_initialize(self):
self.k.initialize()
def test_delete_leader(self):
self.k.delete_leader()
def test_cancel_initialization(self):
self.k.cancel_initialization()
@patch.object(k8s_client.CoreV1Api, 'delete_collection_namespaced_config_map',
Mock(side_effect=k8s_client.rest.ApiException(403, '')))
def test_delete_cluster(self):
self.k.delete_cluster()
@patch('kubernetes.config.load_kube_config', Mock())
@patch.object(k8s_client.CoreV1Api, 'create_namespaced_endpoints',
Mock(side_effect=[k8s_client.rest.ApiException(502, ''), k8s_client.rest.ApiException(500, '')]))
def test_delete_sync_state(self):
k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10,
'labels': {'f': 'b'}, 'use_endpoints': True, 'pod_ip': '10.0.0.0'})
self.assertFalse(k.delete_sync_state())
def test_watch(self):
self.k.set_ttl(10)
self.k.watch(None, 0)
self.k.watch(None, 0)
with patch.object(k8s_watch.Watch, 'stream',
Mock(side_effect=[Exception, [], KeyboardInterrupt,
[{'raw_object': {'metadata': {'resourceVersion': '2'}}}]])):
self.assertFalse(self.k.watch('1', 2))
self.assertRaises(KeyboardInterrupt, self.k.watch, '1', 2)
self.assertTrue(self.k.watch('1', 2))
def test_set_history_value(self):
self.k.set_history_value('{}')
@patch('kubernetes.config.load_kube_config', Mock())
@patch.object(k8s_client.CoreV1Api, 'patch_namespaced_pod', Mock(return_value=True))
@patch.object(k8s_client.CoreV1Api, 'create_namespaced_endpoints', Mock())
@patch.object(k8s_client.CoreV1Api, 'create_namespaced_service',
Mock(side_effect=[True, False, k8s_client.rest.ApiException(500, '')]))
def test__create_config_service(self):
k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10,
'labels': {'f': 'b'}, 'use_endpoints': True, 'pod_ip': '10.0.0.0'})
self.assertIsNotNone(k.patch_or_create_config({'foo': 'bar'}))
self.assertIsNotNone(k.patch_or_create_config({'foo': 'bar'}))
k.touch_member({'state': 'running', 'role': 'replica'})
-36
View File
@@ -1,36 +0,0 @@
import os
import sys
import unittest
import yaml
from mock import Mock, patch
from patroni.config import Config
from patroni.log import PatroniLogger
class TestPatroniLogger(unittest.TestCase):
@patch('logging.FileHandler._open', Mock())
def test_patroni_logger(self):
config = {
'log': {
'dir': 'foo',
'file_size': 4096,
'file_num': 5,
'loggers': {
'foo.bar': 'INFO'
}
},
'restapi': {}, 'postgresql': {'data_dir': 'foo'}
}
sys.argv = ['patroni.py']
os.environ[Config.PATRONI_CONFIG_VARIABLE] = yaml.dump(config, default_flow_style=False)
logger = PatroniLogger()
patroni_config = Config()
logger.reload_config(patroni_config['log'])
self.assertEqual(logger.handler.maxBytes, config['log']['file_size'])
self.assertEqual(logger.handler.backupCount, config['log']['file_num'])
config['log'].pop('dir')
logger.reload_config(config['log'])
+3 -9
View File
@@ -12,7 +12,7 @@ from patroni.exceptions import DCSError
from patroni import Patroni, main as _main, patroni_main
from six.moves import BaseHTTPServer
from test_etcd import SleepException, etcd_read, etcd_write
from test_postgresql import Postgresql, psycopg2_connect, MockPostmaster
from test_postgresql import Postgresql, psycopg2_connect
class MockFrozenImporter(object):
@@ -26,7 +26,7 @@ class MockFrozenImporter(object):
@patch.object(Postgresql, 'write_pg_hba', Mock())
@patch.object(Postgresql, '_write_postgresql_conf', Mock())
@patch.object(Postgresql, 'write_recovery_conf', Mock())
@patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
@patch.object(Postgresql, 'call_nowait', Mock())
@patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
@patch.object(AsyncExecutor, 'run', Mock())
@@ -73,7 +73,7 @@ class TestPatroni(unittest.TestCase):
mock_getpid.return_value = 2
_main()
with patch('sys.frozen', Mock(return_value=True), create=True), patch('os.setsid', Mock()):
with patch('sys.frozen', Mock(return_value=True), create=True):
sys.argv = ['/patroni', 'pg_ctl_start', 'postgres', '-D', '/data', '--max_connections=100']
_main()
@@ -103,9 +103,7 @@ class TestPatroni(unittest.TestCase):
@patch('patroni.config.Config.save_cache', Mock())
@patch('patroni.config.Config.reload_local_configuration', Mock(return_value=True))
@patch('patroni.ha.Ha.is_leader', Mock(return_value=True))
@patch.object(Postgresql, 'state', PropertyMock(return_value='running'))
@patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
def test_run(self):
self.p.postgresql.set_role('replica')
self.p.sighup_handler()
@@ -152,7 +150,3 @@ class TestPatroni(unittest.TestCase):
self.assertTrue(self.p.nosync)
self.p.tags['nosync'] = None
self.assertFalse(self.p.nosync)
def test_shutdown(self):
self.p.api.shutdown = Mock(side_effect=Exception)
self.p.shutdown()
+220 -360
View File
@@ -1,4 +1,4 @@
import datetime
import errno
import mock # for the mock.call method, importing it without a namespace breaks python3
import os
import psycopg2
@@ -8,13 +8,12 @@ import unittest
from mock import Mock, MagicMock, PropertyMock, patch, mock_open
from patroni.async_executor import CriticalTask
from patroni.dcs import Cluster, ClusterConfig, Leader, Member, RemoteMember, SyncState
from patroni.exceptions import PostgresConnectionException, PostgresException
from patroni.dcs import Cluster, Leader, Member, SyncState
from patroni.exceptions import PostgresConnectionException
from patroni.postgresql import Postgresql, STATE_REJECT, STATE_NO_RESPONSE
from patroni.postmaster import PostmasterProcess
from patroni.utils import RetryFailedError
from six.moves import builtins
from threading import Thread, current_thread
from threading import Thread
class MockCursor(object):
@@ -28,20 +27,20 @@ class MockCursor(object):
def execute(self, sql, *params):
if sql.startswith('blabla'):
raise psycopg2.ProgrammingError()
elif sql == 'CHECKPOINT' or sql.startswith('SELECT pg_catalog.pg_create_'):
elif sql == 'CHECKPOINT':
raise psycopg2.OperationalError()
elif sql.startswith('RetryFailedError'):
raise RetryFailedError('retry')
elif sql.startswith('SELECT slot_name'):
self.results = [('blabla', 'physical'), ('foobar', 'physical'), ('ls', 'logical', 'a', 'b')]
elif sql.startswith('SELECT CASE WHEN pg_catalog.pg_is_in_recovery()'):
self.results = [(1, 2)]
elif sql.startswith('SELECT pg_catalog.pg_is_in_recovery()'):
self.results = [(False, 2)]
self.results = [('blabla',), ('foobar',)]
elif sql.startswith('SELECT CASE WHEN pg_is_in_recovery()'):
self.results = [(2,)]
elif sql == 'SELECT pg_is_in_recovery()':
self.results = [(False, )]
elif sql.startswith('WITH replication_info AS ('):
replication_info = '[{"application_name":"walreceiver","client_addr":"1.2.3.4",' +\
'"state":"streaming","sync_state":"async","sync_priority":0}]'
self.results = [('', 0, '', '', '', '', False, replication_info)]
self.results = [('', True, '', '', '', '', False, replication_info)]
elif sql.startswith('SELECT name, setting'):
self.results = [('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
('search_path', 'public', None, 'string', 'user'),
@@ -51,11 +50,6 @@ class MockCursor(object):
('unix_socket_directories', '/tmp', None, 'string', 'postmaster')]
elif sql.startswith('IDENTIFY_SYSTEM'):
self.results = [('1', 2, '0/402EEC0', '')]
elif sql.startswith('SELECT isdir, modification'):
self.results = [(False, datetime.datetime.now())]
elif sql.startswith('SELECT pg_catalog.pg_read_file'):
self.results = [('1\t0/40159C0\tno recovery target specified\n\n' +
'2\t1/40159C0\tno recovery target specified\n',)]
elif sql.startswith('TIMELINE_HISTORY '):
self.results = [('', b'x\t0/40159C0\tno recovery target specified\n\n' +
b'1\t0/40159C0\tno recovery target specified\n\n' +
@@ -101,15 +95,6 @@ class MockConnect(object):
pass
class MockPostmaster(object):
def __init__(self, is_running=True, is_single_master=False):
self.is_running = Mock(return_value=is_running)
self.is_single_master = Mock(return_value=is_single_master)
self.wait_for_user_backends_to_close = Mock()
self.signal_stop = Mock(return_value=None)
self.wait = Mock()
def pg_controldata_string(*args, **kwargs):
return b"""
pg_control version number: 942
@@ -189,7 +174,7 @@ class TestPostgresql(unittest.TestCase):
if not os.path.exists(self.data_dir):
os.makedirs(self.data_dir)
self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': self.data_dir,
'config_dir': self.config_dir, 'retry_timeout': 10, 'pgpass': '/tmp/pgpass0',
'config_dir': self.config_dir, 'retry_timeout': 10,
'listen': '127.0.0.2, 127.0.0.3:5432', 'connect_address': '127.0.0.2:5432',
'authentication': {'superuser': {'username': 'test', 'password': 'test'},
'replication': {'username': 'replicator', 'password': 'rep-pass'}},
@@ -210,11 +195,12 @@ class TestPostgresql(unittest.TestCase):
def tearDown(self):
shutil.rmtree('data')
def test__initdb(self):
self.assertRaises(Exception, self.p.bootstrap, {'initdb': [{'pgdata': 'bar'}]})
self.assertRaises(Exception, self.p.bootstrap, {'initdb': [{'foo': 'bar', 1: 2}]})
self.assertRaises(Exception, self.p.bootstrap, {'initdb': [1]})
self.assertRaises(Exception, self.p.bootstrap, {'initdb': 1})
def test_get_initdb_options(self):
self.assertEquals(self.p.get_initdb_options([{'encoding': 'UTF8'}, 'data-checksums']),
['--encoding=UTF8', '--data-checksums'])
self.assertRaises(Exception, self.p.get_initdb_options, [{'pgdata': 'bar'}])
self.assertRaises(Exception, self.p.get_initdb_options, [{'foo': 'bar', 1: 2}])
self.assertRaises(Exception, self.p.get_initdb_options, [1])
@patch('os.path.exists', Mock(return_value=True))
@patch('os.unlink', Mock())
@@ -226,89 +212,90 @@ class TestPostgresql(unittest.TestCase):
@patch.object(Postgresql, 'wait_for_port_open')
@patch.object(Postgresql, 'is_running')
def test_start(self, mock_is_running, mock_wait_for_port_open, mock_wait_for_startup, mock_popen):
mock_is_running.return_value = MockPostmaster()
mock_is_running.return_value = True
mock_wait_for_port_open.return_value = True
mock_wait_for_startup.return_value = False
mock_popen.return_value.stdout.readline.return_value = '123'
self.assertTrue(self.p.start())
mock_is_running.return_value = None
mock_is_running.return_value = False
open(os.path.join(self.data_dir, 'postmaster.pid'), 'w').close()
pg_conf = os.path.join(self.data_dir, 'postgresql.conf')
open(pg_conf, 'w').close()
self.assertFalse(self.p.start(task=CriticalTask()))
with open(pg_conf) as f:
lines = f.readlines()
self.assertTrue("f.oo = 'bar'\n" in lines)
mock_postmaster = MockPostmaster()
with patch.object(PostmasterProcess, 'start', return_value=mock_postmaster):
pg_conf = os.path.join(self.data_dir, 'postgresql.conf')
open(pg_conf, 'w').close()
self.assertFalse(self.p.start(task=CriticalTask()))
mock_wait_for_startup.return_value = None
self.assertFalse(self.p.start(10))
self.assertIsNone(self.p.start())
with open(pg_conf) as f:
lines = f.readlines()
self.assertTrue("f.oo = 'bar'\n" in lines)
mock_wait_for_startup.return_value = None
self.assertFalse(self.p.start(10))
self.assertIsNone(self.p.start())
mock_wait_for_port_open.return_value = False
self.assertFalse(self.p.start())
task = CriticalTask()
task.cancel()
self.assertFalse(self.p.start(task=task))
self.p.cancel()
mock_wait_for_port_open.return_value = False
self.assertFalse(self.p.start())
task = CriticalTask()
task.cancel()
self.assertFalse(self.p.start(task=task))
@patch.object(Postgresql, 'pg_isready')
@patch.object(Postgresql, 'read_pid_file')
@patch.object(Postgresql, 'is_pid_running')
@patch('patroni.postgresql.polling_loop', Mock(return_value=range(1)))
def test_wait_for_port_open(self, mock_pg_isready):
def test_wait_for_port_open(self, mock_is_pid_running, mock_read_pid_file, mock_pg_isready):
mock_is_pid_running.return_value = False
mock_pg_isready.return_value = STATE_NO_RESPONSE
mock_postmaster = MockPostmaster(is_running=False)
# No pid file and postmaster death
self.assertFalse(self.p.wait_for_port_open(mock_postmaster, 1))
mock_read_pid_file.return_value = {}
self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
mock_postmaster.is_running.return_value = True
mock_is_pid_running.return_value = True
# timeout
self.assertFalse(self.p.wait_for_port_open(mock_postmaster, 1))
mock_read_pid_file.return_value = {'pid', 1}
self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
# Garbage pid
mock_read_pid_file.return_value = {'pid': 'garbage', 'start_time': '101', 'data_dir': '',
'socket_dir': '', 'port': '', 'listen_addr': ''}
self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
# Not ready
mock_read_pid_file.return_value = {'pid': '42', 'start_time': '101', 'data_dir': '',
'socket_dir': '', 'port': '', 'listen_addr': ''}
self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
# pg_isready failure
mock_pg_isready.return_value = 'garbage'
self.assertTrue(self.p.wait_for_port_open(mock_postmaster, 1))
# cancelled
self.p.cancel()
self.assertFalse(self.p.wait_for_port_open(mock_postmaster, 1))
self.assertTrue(self.p.wait_for_port_open(42, 100., 1))
@patch('time.sleep', Mock())
@patch.object(Postgresql, 'is_running')
@patch.object(Postgresql, '_wait_for_connection_close', Mock())
def test_stop(self, mock_is_running):
# Postmaster is not running
@patch.object(Postgresql, 'get_pid')
def test_stop(self, mock_get_pid, mock_is_running):
mock_callback = Mock()
mock_is_running.return_value = None
mock_is_running.return_value = False
self.assertTrue(self.p.stop(on_safepoint=mock_callback))
mock_callback.assert_called()
# Is running, stopped successfully
mock_is_running.return_value = mock_postmaster = MockPostmaster()
mock_is_running.return_value = True
mock_get_pid.return_value = 0
mock_callback.reset_mock()
self.assertTrue(self.p.stop(on_safepoint=mock_callback))
mock_callback.assert_called()
mock_postmaster.signal_stop.assert_called()
# Stop signal failed
mock_postmaster.signal_stop.return_value = False
mock_get_pid.return_value = -1
self.assertFalse(self.p.stop())
# Stop signal failed to find process
mock_postmaster.signal_stop.return_value = True
mock_callback.reset_mock()
self.assertTrue(self.p.stop(on_safepoint=mock_callback))
mock_callback.assert_called()
mock_get_pid.return_value = 123
with patch('os.kill', Mock(side_effect=[OSError(errno.ESRCH, ''), OSError, None])):
self.assertTrue(self.p.stop())
self.assertFalse(self.p.stop())
self.assertTrue(self.p.stop())
with patch.object(Postgresql, '_signal_postmaster_stop', Mock(return_value=(123, None))):
with patch.object(Postgresql, 'is_pid_running', Mock(side_effect=[True, False, False])):
self.assertTrue(self.p.stop())
def test_restart(self):
self.p.start = Mock(return_value=False)
self.assertFalse(self.p.restart())
self.assertEqual(self.p.state, 'restart failed (restarting)')
self.assertEquals(self.p.state, 'restart failed (restarting)')
@patch.object(builtins, 'open', MagicMock())
def test_write_pgpass(self):
@@ -317,18 +304,17 @@ class TestPostgresql(unittest.TestCase):
def test_checkpoint(self):
with patch.object(MockCursor, 'fetchone', Mock(return_value=(True, ))):
self.assertEqual(self.p.checkpoint({'user': 'postgres'}), 'is_in_recovery=true')
self.assertEquals(self.p.checkpoint({'user': 'postgres'}), 'is_in_recovery=true')
with patch.object(MockCursor, 'execute', Mock(return_value=None)):
self.assertIsNone(self.p.checkpoint())
self.assertEqual(self.p.checkpoint(), 'not accessible or not healty')
self.assertEquals(self.p.checkpoint(), 'not accessible or not healty')
@patch.object(Postgresql, 'cancellable_subprocess_call')
@patch('subprocess.call', side_effect=OSError)
@patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
def test_pg_rewind(self, mock_cancellable_subprocess_call):
def test_pg_rewind(self, mock_call):
r = {'user': '', 'host': '', 'port': '', 'database': '', 'password': ''}
mock_cancellable_subprocess_call.return_value = 0
self.assertTrue(self.p.pg_rewind(r))
mock_cancellable_subprocess_call.side_effect = OSError
subprocess.call = mock_call
self.assertFalse(self.p.pg_rewind(r))
def test_check_recovery_conf(self):
@@ -341,46 +327,47 @@ class TestPostgresql(unittest.TestCase):
@patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
def test__get_local_timeline_lsn(self):
self.p.trigger_check_diverged_lsn()
with patch.object(Postgresql, 'controldata', Mock(return_value={'Database cluster state': 'shut down'})):
self.p.rewind_needed_and_possible(self.leader)
with patch.object(Postgresql, 'controldata',
Mock(return_value={'Database cluster state': 'shut down in recovery',
'Minimum recovery ending location': '0/0',
"Min recovery ending loc's timeline": '0'})):
self.p.rewind_or_reinitialize_needed_and_possible(self.leader)
Mock(return_value={'Database cluster state': 'shut down in recovery'})):
self.p.rewind_needed_and_possible(self.leader)
with patch.object(Postgresql, 'is_running', Mock(return_value=True)):
with patch.object(MockCursor, 'fetchone', Mock(side_effect=[(False, ), Exception])):
self.p.rewind_or_reinitialize_needed_and_possible(self.leader)
self.p.rewind_needed_and_possible(self.leader)
@patch.object(Postgresql, 'start', Mock())
@patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
@patch.object(Postgresql, '_get_local_timeline_lsn', Mock(return_value=(2, '40159C1')))
@patch.object(Postgresql, '_get_local_timeline_lsn', Mock(return_value=(2, '0/40159C1')))
@patch.object(Postgresql, 'check_leader_is_not_in_recovery')
def test__check_timeline_and_lsn(self, mock_check_leader_is_not_in_recovery):
mock_check_leader_is_not_in_recovery.return_value = False
self.p.trigger_check_diverged_lsn()
self.assertFalse(self.p.rewind_or_reinitialize_needed_and_possible(self.leader))
self.leader = self.leader.member
self.assertFalse(self.p.rewind_or_reinitialize_needed_and_possible(self.leader))
self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
mock_check_leader_is_not_in_recovery.return_value = True
self.assertFalse(self.p.rewind_or_reinitialize_needed_and_possible(self.leader))
self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
self.p.trigger_check_diverged_lsn()
with patch('psycopg2.connect', Mock(side_effect=Exception)):
self.assertFalse(self.p.rewind_or_reinitialize_needed_and_possible(self.leader))
self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
with patch.object(MockCursor, 'fetchone',
Mock(side_effect=[('', 2, '0/0'), ('', b'2\tG/40159C0\tno recovery target specified\n\n')])):
self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
self.p.trigger_check_diverged_lsn()
with patch.object(MockCursor, 'fetchone', Mock(side_effect=[('', 2, '0/0'), ('', b'3\t0/40159C0\tn\n')])):
self.assertFalse(self.p.rewind_or_reinitialize_needed_and_possible(self.leader))
with patch.object(MockCursor, 'fetchone',
Mock(side_effect=[('', 2, '0/0'), ('', b'3\t040159C0\tno recovery target specified\n')])):
self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
self.p.trigger_check_diverged_lsn()
with patch.object(MockCursor, 'fetchone', Mock(return_value=('', 1, '0/0'))):
with patch.object(Postgresql, '_get_local_timeline_lsn', Mock(return_value=(1, '0/0'))):
self.assertFalse(self.p.rewind_or_reinitialize_needed_and_possible(self.leader))
self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
self.p.trigger_check_diverged_lsn()
self.assertTrue(self.p.rewind_or_reinitialize_needed_and_possible(self.leader))
self.assertTrue(self.p.rewind_needed_and_possible(self.leader))
@patch.object(MockCursor, 'fetchone', Mock(side_effect=[(True,), Exception]))
def test_check_leader_is_not_in_recovery(self):
self.p.check_leader_is_not_in_recovery()
self.p.check_leader_is_not_in_recovery()
@patch.object(Postgresql, 'cancellable_subprocess_call', Mock(return_value=0))
@patch.object(Postgresql, 'checkpoint', side_effect=['', '1'])
@patch.object(Postgresql, 'stop', Mock(return_value=False))
@patch.object(Postgresql, 'start', Mock())
@@ -403,8 +390,7 @@ class TestPostgresql(unittest.TestCase):
@patch.object(Postgresql, 'is_running', Mock(return_value=False))
@patch.object(Postgresql, 'start', Mock())
def test_follow(self):
m = RemoteMember('1', {'restore_command': '2', 'recovery_min_apply_delay': 3, 'archive_cleanup_command': '4'})
self.p.follow(m)
self.p.follow(None)
@patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string))
def test_can_rewind(self):
@@ -418,111 +404,45 @@ class TestPostgresql(unittest.TestCase):
self.assertFalse(self.p.can_rewind)
@patch('time.sleep', Mock())
@patch.object(Postgresql, 'cancellable_subprocess_call')
@patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True))
def test_create_replica(self, mock_cancellable_subprocess_call):
self.p.delete_trigger_file = Mock(side_effect=OSError)
self.p.config['create_replica_methods'] = ['pgBackRest']
self.p.config['pgBackRest'] = {'command': 'pgBackRest', 'keep_data': True, 'no_params': True}
mock_cancellable_subprocess_call.return_value = 0
self.assertEqual(self.p.create_replica(self.leader), 0)
self.p.config['create_replica_methods'] = ['wale', 'basebackup']
self.p.config['wale'] = {'command': 'foo'}
self.assertEqual(self.p.create_replica(self.leader), 0)
del self.p.config['wale']
self.assertEqual(self.p.create_replica(self.leader), 0)
self.p.config['create_replica_methods'] = ['basebackup']
self.p.config['basebackup'] = [{'max_rate': '100M'}, 'no-sync']
self.assertEqual(self.p.create_replica(self.leader), 0)
self.p.config['basebackup'] = [{'max_rate': '100M', 'compress': '9'}]
with mock.patch('patroni.postgresql.logger.error', new_callable=Mock()) as mock_logger:
self.p.create_replica(self.leader)
mock_logger.assert_called_once()
self.assertTrue("only one key-value is allowed and value should be a string" in mock_logger.call_args[0][0],
"not matching {0}".format(mock_logger.call_args[0][0]))
self.p.config['basebackup'] = [42]
with mock.patch('patroni.postgresql.logger.error', new_callable=Mock()) as mock_logger:
self.p.create_replica(self.leader)
mock_logger.assert_called_once()
self.assertTrue("value should be string value or a single key-value pair" in mock_logger.call_args[0][0],
"not matching {0}".format(mock_logger.call_args[0][0]))
self.p.config['basebackup'] = {"foo": "bar"}
self.assertEqual(self.p.create_replica(self.leader), 0)
self.p.config['create_replica_methods'] = ['wale', 'basebackup']
del self.p.config['basebackup']
mock_cancellable_subprocess_call.return_value = 1
self.assertEqual(self.p.create_replica(self.leader), 1)
mock_cancellable_subprocess_call.side_effect = Exception('foo')
self.assertEqual(self.p.create_replica(self.leader), 1)
mock_cancellable_subprocess_call.side_effect = [1, 0]
self.assertEqual(self.p.create_replica(self.leader), 0)
mock_cancellable_subprocess_call.side_effect = [Exception(), 0]
self.assertEqual(self.p.create_replica(self.leader), 0)
self.p.cancel()
self.assertEqual(self.p.create_replica(self.leader), 1)
@patch('time.sleep', Mock())
@patch.object(Postgresql, 'cancellable_subprocess_call')
@patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True))
def test_create_replica_old_format(self, mock_cancellable_subprocess_call):
""" The same test as before but with old 'create_replica_method'
to test backward compatibility
"""
def test_create_replica(self):
self.p.delete_trigger_file = Mock(side_effect=OSError)
with patch('subprocess.call', Mock(side_effect=[1, 0])):
self.assertEquals(self.p.create_replica(self.leader), 0)
with patch('subprocess.call', Mock(side_effect=[Exception(), 0])):
self.assertEquals(self.p.create_replica(self.leader), 0)
self.p.config['create_replica_method'] = ['wale', 'basebackup']
self.p.config['wale'] = {'command': 'foo'}
mock_cancellable_subprocess_call.return_value = 0
self.assertEqual(self.p.create_replica(self.leader), 0)
del self.p.config['wale']
self.assertEqual(self.p.create_replica(self.leader), 0)
with patch('subprocess.call', Mock(return_value=0)):
self.assertEquals(self.p.create_replica(self.leader), 0)
del self.p.config['wale']
self.assertEquals(self.p.create_replica(self.leader), 0)
self.p.config['create_replica_method'] = ['basebackup']
self.p.config['basebackup'] = [{'max_rate': '100M'}, 'no-sync']
self.assertEqual(self.p.create_replica(self.leader), 0)
with patch('subprocess.call', Mock(side_effect=Exception("foo"))):
self.assertEquals(self.p.create_replica(self.leader), 1)
self.p.config['create_replica_method'] = ['wale', 'basebackup']
del self.p.config['basebackup']
mock_cancellable_subprocess_call.return_value = 1
self.assertEqual(self.p.create_replica(self.leader), 1)
def test_basebackup(self):
self.p.cancel()
self.p.basebackup(None, None, {'foo': 'bar'})
with patch('subprocess.call', Mock(return_value=1)):
self.assertEquals(self.p.create_replica(self.leader), 1)
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
def test_sync_replication_slots(self):
self.p.start()
config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'},
'A': 0, 'test_3': 0, 'b': {'type': 'logical', 'plugin': '1'}}}, 1)
cluster = Cluster(True, config, self.leader, 0, [self.me, self.other, self.leadermem], None, None, None)
cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None, None)
with mock.patch('patroni.postgresql.Postgresql._query', Mock(side_effect=psycopg2.OperationalError)):
self.p.sync_replication_slots(cluster)
self.p.sync_replication_slots(cluster)
with mock.patch('patroni.postgresql.Postgresql.role', new_callable=PropertyMock(return_value='replica')):
self.p.sync_replication_slots(cluster)
with patch.object(Postgresql, 'drop_replication_slot', Mock(return_value=True)),\
patch('patroni.dcs.logger.error', new_callable=Mock()) as errorlog_mock:
with mock.patch('patroni.postgresql.logger.error', new_callable=Mock()) as errorlog_mock:
self.p.query = Mock()
alias1 = Member(0, 'test-3', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres'})
alias2 = Member(0, 'test.3', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres'})
cluster.members.extend([alias1, alias2])
self.p.sync_replication_slots(cluster)
self.assertEqual(errorlog_mock.call_count, 5)
ca = errorlog_mock.call_args_list[0][0][1]
self.assertTrue("test-3" in ca, "non matching {0}".format(ca))
self.assertTrue("test.3" in ca, "non matching {0}".format(ca))
errorlog_mock.assert_called_once()
assert "test-3" in errorlog_mock.call_args[0][1]
assert "test.3" in errorlog_mock.call_args[0][1]
@patch.object(MockCursor, 'execute', Mock(side_effect=psycopg2.OperationalError))
def test__query(self):
@@ -538,7 +458,6 @@ class TestPostgresql(unittest.TestCase):
@patch.object(Postgresql, 'pg_isready', Mock(return_value=STATE_REJECT))
def test_is_leader(self):
self.assertTrue(self.p.is_leader())
self.p.reset_cluster_info_state()
with patch.object(Postgresql, '_query', Mock(side_effect=RetryFailedError(''))):
self.assertRaises(PostgresConnectionException, self.p.is_leader)
@@ -554,29 +473,21 @@ class TestPostgresql(unittest.TestCase):
def test_promote(self):
self.p.set_role('replica')
self.assertIsNone(self.p.promote(0))
self.assertTrue(self.p.promote(0))
self.assertTrue(self.p.promote())
self.assertTrue(self.p.promote())
def test_timeline_wal_position(self):
self.assertEqual(self.p.timeline_wal_position(), (1, 2))
Thread(target=self.p.timeline_wal_position).start()
def test_last_operation(self):
self.assertEquals(self.p.last_operation(), '2')
Thread(target=self.p.last_operation).start()
@patch.object(PostmasterProcess, 'from_pidfile')
def test_is_running(self, mock_frompidfile):
# Cached postmaster running
mock_postmaster = self.p._postmaster_proc = MockPostmaster()
self.assertEqual(self.p.is_running(), mock_postmaster)
# Cached postmaster not running, no postmaster running
mock_postmaster.is_running.return_value = False
mock_frompidfile.return_value = None
self.assertEqual(self.p.is_running(), None)
self.assertEqual(self.p._postmaster_proc, None)
# No cached postmaster, postmaster running
mock_frompidfile.return_value = mock_postmaster2 = MockPostmaster()
self.assertEqual(self.p.is_running(), mock_postmaster2)
self.assertEqual(self.p._postmaster_proc, mock_postmaster2)
@patch('os.path.isfile', Mock(return_value=True))
@patch('os.kill', Mock(side_effect=Exception))
@patch('os.getpid', Mock(return_value=2))
@patch('os.getppid', Mock(return_value=2))
@patch.object(builtins, 'open', mock_open(read_data='-1'))
@patch.object(Postgresql, '_version_file_exists', Mock(return_value=True))
def test_is_running(self):
self.assertFalse(self.p.is_running())
@patch('shlex.split', Mock(side_effect=OSError))
def test_call_nowait(self):
@@ -588,7 +499,7 @@ class TestPostgresql(unittest.TestCase):
def test_non_existing_callback(self):
self.assertFalse(self.p.call_nowait('foobar'))
@patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
def test_is_leader_exception(self):
self.p.start()
self.p.query = Mock(side_effect=psycopg2.OperationalError("not supported"))
@@ -623,16 +534,14 @@ class TestPostgresql(unittest.TestCase):
lines = f.readlines()
self.assertTrue('host replication replicator 127.0.0.1/32 md5\n' in lines)
@patch.object(Postgresql, 'cancellable_subprocess_call')
def test_custom_bootstrap(self, mock_cancellable_subprocess_call):
self.p.config.pop('pg_hba')
def test_custom_bootstrap(self):
config = {'method': 'foo', 'foo': {'command': 'bar'}}
mock_cancellable_subprocess_call.return_value = 1
self.assertFalse(self.p.bootstrap(config))
mock_cancellable_subprocess_call.return_value = 0
with patch('subprocess.Popen', Mock(side_effect=Exception("42"))),\
with patch('subprocess.call', Mock(return_value=1)):
self.assertFalse(self.p.bootstrap(config))
with patch('subprocess.call', Mock(side_effect=Exception)):
self.assertFalse(self.p.bootstrap(config))
with patch('subprocess.call', Mock(return_value=0)),\
patch('subprocess.Popen', Mock(side_effect=Exception("42"))),\
patch('os.path.isfile', Mock(return_value=True)),\
patch('os.unlink', Mock()),\
patch.object(Postgresql, 'save_configuration_files', Mock()),\
@@ -648,12 +557,8 @@ class TestPostgresql(unittest.TestCase):
self.p.bootstrap(config)
self.assertEqual(str(e.exception), '42')
mock_cancellable_subprocess_call.side_effect = Exception
self.assertFalse(self.p.bootstrap(config))
@patch('time.sleep', Mock())
@patch('os.unlink', Mock())
@patch('os.path.isfile', Mock(return_value=True))
@patch.object(Postgresql, 'run_bootstrap_post_init', Mock(return_value=True))
@patch.object(Postgresql, '_custom_bootstrap', Mock(return_value=True))
@patch.object(Postgresql, 'start', Mock(return_value=True))
@@ -670,12 +575,6 @@ class TestPostgresql(unittest.TestCase):
self.p.post_bootstrap({}, task)
self.assertTrue(task.result)
self.p.bootstrap(config)
with patch.object(Postgresql, 'pending_restart', PropertyMock(return_value=True)), \
patch.object(Postgresql, 'restart', Mock()) as mock_restart:
self.p.post_bootstrap({}, task)
mock_restart.assert_called_once()
self.p.bootstrap(config)
self.p.set_state('stopped')
self.p.reload_config({'authentication': {'superuser': {'username': 'p', 'password': 'p'},
@@ -685,27 +584,26 @@ class TestPostgresql(unittest.TestCase):
self.p.post_bootstrap({}, task)
mock_restart.assert_called_once()
@patch.object(Postgresql, 'cancellable_subprocess_call')
def test_run_bootstrap_post_init(self, mock_cancellable_subprocess_call):
mock_cancellable_subprocess_call.return_value = 1
self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
def test_run_bootstrap_post_init(self):
with patch('subprocess.call', Mock(return_value=1)):
self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
mock_cancellable_subprocess_call.return_value = 0
self.p._superuser.pop('username')
self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
mock_cancellable_subprocess_call.assert_called()
args, kwargs = mock_cancellable_subprocess_call.call_args
self.assertTrue('PGPASSFILE' in kwargs['env'])
self.assertEqual(args[0], ['/bin/false', 'postgres://127.0.0.2:5432/postgres'])
with patch('subprocess.call', Mock(side_effect=OSError)):
self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
mock_cancellable_subprocess_call.reset_mock()
self.p._local_address.pop('host')
self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
mock_cancellable_subprocess_call.assert_called()
self.assertEqual(mock_cancellable_subprocess_call.call_args[0][0], ['/bin/false', 'postgres://:5432/postgres'])
with patch('subprocess.call', Mock(return_value=0)) as mock_method:
self.p._superuser.pop('username')
self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
mock_method.assert_called()
args, kwargs = mock_method.call_args
self.assertTrue('PGPASSFILE' in kwargs['env'])
self.assertEquals(args[0], ['/bin/false', 'postgres://127.0.0.2:5432/postgres'])
mock_cancellable_subprocess_call.side_effect = OSError
self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
mock_method.reset_mock()
self.p._local_address.pop('host')
self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
mock_method.assert_called()
self.assertEquals(mock_method.call_args[0][0], ['/bin/false', 'postgres://:5432/postgres'])
@patch('patroni.postgresql.Postgresql.create_replica', Mock(return_value=0))
def test_clone(self):
@@ -714,7 +612,7 @@ class TestPostgresql(unittest.TestCase):
@patch('os.listdir', Mock(return_value=['recovery.conf']))
@patch('os.path.exists', Mock(return_value=True))
def test_get_postgres_role_from_data_directory(self):
self.assertEqual(self.p.get_postgres_role_from_data_directory(), 'replica')
self.assertEquals(self.p.get_postgres_role_from_data_directory(), 'replica')
def test_remove_data_directory(self):
self.p.remove_data_directory()
@@ -729,13 +627,13 @@ class TestPostgresql(unittest.TestCase):
def test_controldata(self):
with patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string)):
data = self.p.controldata()
self.assertEqual(len(data), 50)
self.assertEqual(data['Database cluster state'], 'shut down in recovery')
self.assertEqual(data['wal_log_hints setting'], 'on')
self.assertEqual(int(data['Database block size']), 8192)
self.assertEquals(len(data), 50)
self.assertEquals(data['Database cluster state'], 'shut down in recovery')
self.assertEquals(data['wal_log_hints setting'], 'on')
self.assertEquals(int(data['Database block size']), 8192)
with patch('subprocess.check_output', Mock(side_effect=subprocess.CalledProcessError(1, ''))):
self.assertEqual(self.p.controldata(), {})
self.assertEquals(self.p.controldata(), {})
@patch('patroni.postgresql.Postgresql._version_file_exists', Mock(return_value=True))
@patch('subprocess.check_output', MagicMock(return_value=0, side_effect=pg_controldata_string))
@@ -789,9 +687,9 @@ class TestPostgresql(unittest.TestCase):
@patch.object(Postgresql, '_version_file_exists', Mock(return_value=True))
def test_get_major_version(self):
with patch.object(builtins, 'open', mock_open(read_data='9.4')):
self.assertEqual(self.p.get_major_version(), 90400)
self.assertEquals(self.p.get_major_version(), 90400)
with patch.object(builtins, 'open', Mock(side_effect=Exception)):
self.assertEqual(self.p.get_major_version(), 0)
self.assertEquals(self.p.get_major_version(), 0)
def test_postmaster_start_time(self):
with patch.object(MockCursor, "fetchone", Mock(return_value=('foo', True, '', '', '', '', False))):
@@ -803,40 +701,38 @@ class TestPostgresql(unittest.TestCase):
with patch('subprocess.call', return_value=0):
self.p._state = 'starting'
self.assertFalse(self.p.check_for_startup())
self.assertEqual(self.p.state, 'running')
self.assertEquals(self.p.state, 'running')
with patch('subprocess.call', return_value=1):
self.p._state = 'starting'
self.assertTrue(self.p.check_for_startup())
self.assertEqual(self.p.state, 'starting')
self.assertEquals(self.p.state, 'starting')
with patch('subprocess.call', return_value=2):
self.p._state = 'starting'
self.assertFalse(self.p.check_for_startup())
self.assertEqual(self.p.state, 'start failed')
self.assertEquals(self.p.state, 'start failed')
with patch('subprocess.call', return_value=0):
self.p._state = 'running'
self.assertFalse(self.p.check_for_startup())
self.assertEqual(self.p.state, 'running')
self.assertEquals(self.p.state, 'running')
with patch('subprocess.call', return_value=127):
self.p._state = 'running'
self.assertFalse(self.p.check_for_startup())
self.assertEqual(self.p.state, 'running')
self.assertEquals(self.p.state, 'running')
self.p._state = 'starting'
self.assertFalse(self.p.check_for_startup())
self.assertEqual(self.p.state, 'running')
self.assertEquals(self.p.state, 'running')
def test_wait_for_startup(self):
state = {'sleeps': 0, 'num_rejects': 0, 'final_return': 0}
self.__thread_ident = current_thread().ident
def increment_sleeps(*args):
if current_thread().ident == self.__thread_ident:
print("Sleep")
state['sleeps'] += 1
print("Sleep")
state['sleeps'] += 1
def isready_return(*args):
ret = 1 if state['sleeps'] < state['num_rejects'] else state['final_return']
@@ -852,12 +748,12 @@ class TestPostgresql(unittest.TestCase):
self.p._state = 'stopped'
self.assertTrue(self.p.wait_for_startup())
self.assertEqual(state['sleeps'], 0)
self.assertEquals(state['sleeps'], 0)
self.p._state = 'starting'
state['num_rejects'] = 5
self.assertTrue(self.p.wait_for_startup())
self.assertEqual(state['sleeps'], 5)
self.assertEquals(state['sleeps'], 5)
self.p._state = 'starting'
state['sleeps'] = 0
@@ -868,46 +764,54 @@ class TestPostgresql(unittest.TestCase):
state['sleeps'] = 0
state['final_return'] = 0
self.assertFalse(self.p.wait_for_startup(timeout=2))
self.assertEqual(state['sleeps'], 3)
self.assertEquals(state['sleeps'], 3)
with patch.object(Postgresql, 'check_startup_state_changed', Mock(return_value=False)):
self.p.cancel()
self.p._state = 'starting'
self.assertIsNone(self.p.wait_for_startup())
def test_read_pid_file(self):
pidfile = os.path.join(self.data_dir, 'postmaster.pid')
if os.path.exists(pidfile):
os.remove(pidfile)
self.assertEquals(self.p.read_pid_file(), {})
@patch('os.kill')
def test_is_pid_running(self, mock_kill):
mock_kill.return_value = True
self.assertTrue(self.p.is_pid_running(-100))
self.assertFalse(self.p.is_pid_running(0))
self.assertFalse(self.p.is_pid_running(None))
def test_pick_sync_standby(self):
cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None,
SyncState(0, self.me.name, self.leadermem.name), None)
SyncState(0, self.me.name, self.leadermem.name))
with patch.object(Postgresql, "query", return_value=[
(self.leadermem.name, 'streaming', 'sync'),
(self.me.name, 'streaming', 'async'),
(self.other.name, 'streaming', 'async'),
]):
self.assertEqual(self.p.pick_synchronous_standby(cluster), (self.leadermem.name, True))
self.assertEquals(self.p.pick_synchronous_standby(cluster), (self.leadermem.name, True))
with patch.object(Postgresql, "query", return_value=[
(self.me.name, 'streaming', 'async'),
(self.leadermem.name, 'streaming', 'potential'),
(self.other.name, 'streaming', 'async'),
]):
self.assertEqual(self.p.pick_synchronous_standby(cluster), (self.leadermem.name, False))
self.assertEquals(self.p.pick_synchronous_standby(cluster), (self.leadermem.name, False))
with patch.object(Postgresql, "query", return_value=[
(self.me.name, 'streaming', 'async'),
(self.other.name, 'streaming', 'async'),
]):
self.assertEqual(self.p.pick_synchronous_standby(cluster), (self.me.name, False))
self.assertEquals(self.p.pick_synchronous_standby(cluster), (self.me.name, False))
with patch.object(Postgresql, "query", return_value=[
('missing', 'streaming', 'sync'),
(self.me.name, 'streaming', 'async'),
(self.other.name, 'streaming', 'async'),
]):
self.assertEqual(self.p.pick_synchronous_standby(cluster), (self.me.name, False))
self.assertEquals(self.p.pick_synchronous_standby(cluster), (self.me.name, False))
with patch.object(Postgresql, "query", return_value=[]):
self.assertEqual(self.p.pick_synchronous_standby(cluster), (None, False))
self.assertEquals(self.p.pick_synchronous_standby(cluster), (None, False))
def test_set_sync_standby(self):
def value_in_conf():
@@ -918,22 +822,22 @@ class TestPostgresql(unittest.TestCase):
mock_reload = self.p.reload = Mock()
self.p.set_synchronous_standby('n1')
self.assertEqual(value_in_conf(), "synchronous_standby_names = 'n1'")
self.assertEquals(value_in_conf(), "synchronous_standby_names = 'n1'")
mock_reload.assert_called()
mock_reload.reset_mock()
self.p.set_synchronous_standby('n1')
mock_reload.assert_not_called()
self.assertEqual(value_in_conf(), "synchronous_standby_names = 'n1'")
self.assertEquals(value_in_conf(), "synchronous_standby_names = 'n1'")
self.p.set_synchronous_standby('n2')
mock_reload.assert_called()
self.assertEqual(value_in_conf(), "synchronous_standby_names = 'n2'")
self.assertEquals(value_in_conf(), "synchronous_standby_names = 'n2'")
mock_reload.reset_mock()
self.p.set_synchronous_standby(None)
mock_reload.assert_called()
self.assertEqual(value_in_conf(), None)
self.assertEquals(value_in_conf(), None)
def test_get_server_parameters(self):
config = {'synchronous_mode': True, 'parameters': {'wal_level': 'hot_standby'}, 'listen': '0'}
@@ -943,83 +847,39 @@ class TestPostgresql(unittest.TestCase):
self.p.set_synchronous_standby('foo')
self.p.get_server_parameters(config)
@patch.object(Postgresql, 'read_pid_file', Mock(return_value={'pid': 'z'}))
def test_get_pid(self):
self.p.get_pid()
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
@patch.object(Postgresql, '_signal_postmaster_stop', Mock(return_value=(123, None)))
@patch.object(Postgresql, 'get_pid', Mock(return_value=123))
@patch('time.sleep', Mock())
def test__wait_for_connection_close(self):
mock_postmaster = MockPostmaster()
with patch.object(Postgresql, 'is_running', Mock(return_value=mock_postmaster)):
mock_postmaster.is_running.side_effect = [True, False, False]
mock_callback = Mock()
@patch.object(Postgresql, 'is_pid_running')
def test__wait_for_connection_close(self, mock_is_pid_running):
mock_is_pid_running.side_effect = [True, False, False]
mock_callback = Mock()
self.p.stop(on_safepoint=mock_callback)
mock_is_pid_running.side_effect = [True, False, False]
with patch.object(MockCursor, "execute", Mock(side_effect=psycopg2.Error)):
self.p.stop(on_safepoint=mock_callback)
mock_postmaster.is_running.side_effect = [True, False, False]
with patch.object(MockCursor, "execute", Mock(side_effect=psycopg2.Error)):
self.p.stop(on_safepoint=mock_callback)
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
@patch.object(Postgresql, '_signal_postmaster_stop', Mock(return_value=(123, None)))
@patch.object(Postgresql, 'get_pid', Mock(return_value=123))
@patch.object(Postgresql, 'is_pid_running', Mock(return_value=False))
@patch('psutil.Process')
def test__wait_for_user_backends_to_close(self, mock_psutil):
child = Mock()
child.cmdline.return_value = ['foo']
mock_psutil.return_value.children.return_value = [child]
mock_callback = Mock()
self.p.stop(on_safepoint=mock_callback)
@patch('os.kill', Mock(side_effect=[OSError(errno.ESRCH, ''), OSError]))
@patch('time.sleep', Mock())
@patch.object(Postgresql, 'is_pid_running', Mock(side_effect=[True, False]))
def test_terminate_starting_postmaster(self):
mock_postmaster = MockPostmaster()
self.p.terminate_starting_postmaster(mock_postmaster)
mock_postmaster.signal_stop.assert_called()
mock_postmaster.wait.assert_called()
def test_read_postmaster_opts(self):
m = mock_open(read_data='/usr/lib/postgres/9.6/bin/postgres "-D" "data/postgresql0" \
"--listen_addresses=127.0.0.1" "--port=5432" "--hot_standby=on" "--wal_level=hot_standby" \
"--wal_log_hints=on" "--max_wal_senders=5" "--max_replication_slots=5"\n')
with patch.object(builtins, 'open', m):
data = self.p.read_postmaster_opts()
self.assertEqual(data['wal_level'], 'hot_standby')
self.assertEqual(int(data['max_replication_slots']), 5)
self.assertEqual(data.get('D'), None)
m.side_effect = IOError
data = self.p.read_postmaster_opts()
self.assertEqual(data, dict())
@patch('subprocess.Popen')
def test_single_user_mode(self, subprocess_popen_mock):
subprocess_popen_mock.return_value.wait.return_value = 0
self.assertEqual(self.p.single_user_mode('CHECKPOINT', {'archive_mode': 'on'}), 0)
@patch('os.listdir', Mock(side_effect=[OSError, ['a', 'b']]))
@patch('os.unlink', Mock(side_effect=OSError))
@patch('os.remove', Mock())
@patch('os.path.islink', Mock(side_effect=[True, False]))
@patch('os.path.isfile', Mock(return_value=True))
def test_cleanup_archive_status(self):
self.p.cleanup_archive_status()
self.p.cleanup_archive_status()
@patch('os.unlink', Mock())
@patch('os.path.isfile', Mock(return_value=True))
@patch.object(Postgresql, 'single_user_mode', Mock(return_value=0))
def test_fix_cluster_state(self):
self.assertTrue(self.p.fix_cluster_state())
def test_replica_cached_timeline(self):
self.assertEqual(self.p.replica_cached_timeline(1), 2)
def test_get_master_timeline(self):
self.assertEqual(self.p.get_master_timeline(), 1)
def test_cancellable_subprocess_call(self):
self.p.cancel()
self.assertRaises(PostgresException, self.p.cancellable_subprocess_call, communicate_input=None)
@patch('patroni.postgresql.polling_loop', Mock(return_value=[0, 0]))
def test_cancel(self):
self.p._cancellable = Mock()
self.p._cancellable.returncode = None
self.p.cancel()
type(self.p._cancellable).returncode = PropertyMock(side_effect=[None, -15])
self.p.cancel()
@patch.object(Postgresql, 'get_postgres_role_from_data_directory', Mock(return_value='replica'))
def test__build_effective_configuration(self):
with patch.object(Postgresql, 'controldata',
Mock(return_value={'max_connections setting': '200',
'max_worker_processes setting': '20',
'max_prepared_xacts setting': '100',
'max_locks_per_xact setting': '100'})):
self.p.cancel()
self.assertFalse(self.p.start())
self.assertTrue(self.p.pending_restart)
self.p.terminate_starting_postmaster(123)
self.p.terminate_starting_postmaster(123)
-102
View File
@@ -1,102 +0,0 @@
import psutil
import unittest
from mock import Mock, patch, mock_open
from patroni.postmaster import PostmasterProcess
from six.moves import builtins
class TestPostmasterProcess(unittest.TestCase):
@patch('psutil.Process.__init__', Mock())
def test_init(self):
proc = PostmasterProcess(-123)
self.assertTrue(proc.is_single_user)
@patch('psutil.Process.create_time')
@patch('psutil.Process.__init__')
@patch('patroni.postmaster.PostmasterProcess._read_postmaster_pidfile')
def test_from_pidfile(self, mock_read, mock_init, mock_create_time):
mock_init.side_effect = psutil.NoSuchProcess(123)
mock_read.return_value = {}
self.assertIsNone(PostmasterProcess.from_pidfile(''))
mock_read.return_value = {"pid": "foo"}
self.assertIsNone(PostmasterProcess.from_pidfile(''))
mock_read.return_value = {"pid": "123"}
self.assertIsNone(PostmasterProcess.from_pidfile(''))
mock_init.side_effect = None
with patch.object(psutil.Process, 'pid', 123), \
patch.object(psutil.Process, 'ppid', return_value=124), \
patch('os.getpid', return_value=125) as mock_ospid, \
patch('os.getppid', return_value=126):
self.assertIsNotNone(PostmasterProcess.from_pidfile(''))
mock_create_time.return_value = 100000
mock_read.return_value = {"pid": "123", "start_time": "200000"}
self.assertIsNone(PostmasterProcess.from_pidfile(''))
mock_read.return_value = {"pid": "123", "start_time": "foobar"}
self.assertIsNotNone(PostmasterProcess.from_pidfile(''))
mock_ospid.return_value = 123
mock_read.return_value = {"pid": "123", "start_time": "100000"}
self.assertIsNone(PostmasterProcess.from_pidfile(''))
@patch('psutil.Process.__init__')
def test_from_pid(self, mock_init):
mock_init.side_effect = psutil.NoSuchProcess(123)
self.assertEqual(PostmasterProcess.from_pid(123), None)
mock_init.side_effect = None
self.assertNotEqual(PostmasterProcess.from_pid(123), None)
@patch('psutil.Process.__init__', Mock())
@patch('psutil.Process.send_signal')
@patch('psutil.Process.pid', Mock(return_value=123))
def test_signal_stop(self, mock_send_signal):
proc = PostmasterProcess(-123)
self.assertEqual(proc.signal_stop('immediate'), False)
mock_send_signal.side_effect = [None, psutil.NoSuchProcess(123), psutil.AccessDenied()]
proc = PostmasterProcess(123)
self.assertEqual(proc.signal_stop('immediate'), None)
self.assertEqual(proc.signal_stop('immediate'), True)
self.assertEqual(proc.signal_stop('immediate'), False)
@patch('psutil.Process.__init__', Mock())
@patch('psutil.wait_procs')
def test_wait_for_user_backends_to_close(self, mock_wait):
c1 = Mock()
c1.cmdline = Mock(return_value=["postgres: startup process "])
c2 = Mock()
c2.cmdline = Mock(return_value=["postgres: postgres postgres [local] idle"])
c3 = Mock()
c3.cmdline = Mock(side_effect=psutil.NoSuchProcess(123))
with patch('psutil.Process.children', Mock(return_value=[c1, c2, c3])):
proc = PostmasterProcess(123)
self.assertIsNone(proc.wait_for_user_backends_to_close())
mock_wait.assert_called_with([c2])
with patch('psutil.Process.children', Mock(side_effect=psutil.NoSuchProcess(123))):
proc = PostmasterProcess(123)
self.assertIsNone(proc.wait_for_user_backends_to_close())
@patch('subprocess.Popen')
@patch.object(PostmasterProcess, 'from_pid')
@patch.object(PostmasterProcess, '_from_pidfile')
def test_start(self, mock_frompidfile, mock_frompid, mock_popen):
mock_frompidfile.return_value._is_postmaster_process.return_value = False
mock_frompid.return_value = "proc 123"
mock_popen.return_value.stdout.readline.return_value = '123'
self.assertEqual(PostmasterProcess.start('true', '/tmp', '/tmp/test.conf', []), "proc 123")
mock_frompid.assert_called_with(123)
mock_frompidfile.side_effect = psutil.NoSuchProcess(123)
self.assertEqual(PostmasterProcess.start('true', '/tmp', '/tmp/test.conf', []), "proc 123")
@patch('psutil.Process.__init__', Mock(side_effect=psutil.NoSuchProcess(123)))
def test_read_postmaster_pidfile(self):
with patch.object(builtins, 'open', Mock(side_effect=IOError)):
self.assertIsNone(PostmasterProcess.from_pidfile(''))
with patch.object(builtins, 'open', mock_open(read_data='123\n')):
self.assertIsNone(PostmasterProcess.from_pidfile(''))
+5 -5
View File
@@ -8,7 +8,7 @@ from patroni.utils import Retry, RetryFailedError, polling_loop
class TestUtils(unittest.TestCase):
def test_polling_loop(self):
self.assertEqual(list(polling_loop(0.001, interval=0.001)), [0])
self.assertEquals(list(polling_loop(0.001, interval=0.001)), [0])
@patch('time.sleep', Mock())
@@ -29,21 +29,21 @@ class TestRetrySleeper(unittest.TestCase):
def test_reset(self):
retry = Retry(delay=0, max_tries=2)
retry(self._fail())
self.assertEqual(retry._attempts, 1)
self.assertEquals(retry._attempts, 1)
retry.reset()
self.assertEqual(retry._attempts, 0)
self.assertEquals(retry._attempts, 0)
def test_too_many_tries(self):
retry = Retry(delay=0)
self.assertRaises(RetryFailedError, retry, self._fail(times=999))
self.assertEqual(retry._attempts, 1)
self.assertEquals(retry._attempts, 1)
def test_maximum_delay(self):
retry = Retry(delay=10, max_tries=100)
retry(self._fail(times=10))
self.assertTrue(retry._cur_delay < 4000, retry._cur_delay)
# gevent's sleep function is picky about the type
self.assertEqual(type(retry._cur_delay), float)
self.assertEquals(type(retry._cur_delay), float)
def test_deadline(self):
retry = Retry(deadline=0.0001)
+7 -20
View File
@@ -7,7 +7,6 @@ from patroni.scripts import wale_restore
from patroni.scripts.wale_restore import WALERestore, main as _main, get_major_version
from six.moves import builtins
from test_postgresql import MockConnect, psycopg2_connect
from threading import current_thread
wale_output_header = (
@@ -43,13 +42,6 @@ class TestWALERestore(unittest.TestCase):
'/etc', 100, 100, 1, 0, WALE_TEST_RETRIES)
def test_should_use_s3_to_create_replica(self):
self.__thread_ident = current_thread().ident
sleeps = [0]
def mock_sleep(*args):
if current_thread().ident == self.__thread_ident:
sleeps[0] += 1
self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
with patch.object(MockConnect, 'server_version', PropertyMock(return_value=100000)):
self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
@@ -63,11 +55,13 @@ class TestWALERestore(unittest.TestCase):
self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
with patch('time.sleep', mock_sleep):
with patch('time.sleep', Mock(return_value=None)) as mock_sleep:
self.wale_restore.no_master = 1
self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
assert self.wale_restore.should_use_s3_to_create_replica()
# verify retries
self.assertEqual(sleeps[0], WALE_TEST_RETRIES)
mock_sleep.assert_has_calls(
[((wale_restore.RETRY_SLEEP_INTERVAL,),)] * WALE_TEST_RETRIES
)
self.wale_restore.master_connection = ''
self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
@@ -110,20 +104,13 @@ class TestWALERestore(unittest.TestCase):
@patch('sys.exit', Mock())
def test_main(self):
self.__thread_ident = current_thread().ident
sleeps = [0]
def mock_sleep(*args):
if current_thread().ident == self.__thread_ident:
sleeps[0] += 1
with patch.object(WALERestore, 'run', Mock(return_value=0)):
self.assertEqual(_main(), 0)
with patch.object(WALERestore, 'run', Mock(return_value=1)), \
patch('time.sleep', mock_sleep):
patch('time.sleep', Mock(return_value=None)) as mock_sleep:
self.assertEqual(_main(), 1)
self.assertTrue(sleeps[0], WALE_TEST_RETRIES)
assert mock_sleep.call_count == WALE_TEST_RETRIES
@patch('os.path.isfile', Mock(return_value=True))
def test_get_major_version(self):
+13 -24
View File
@@ -73,14 +73,14 @@ class TestWatchdog(unittest.TestCase):
@patch.object(LinuxWatchdogDevice, 'can_be_disabled', PropertyMock(return_value=True))
def test_unsafe_timeout_disable_watchdog_and_exit(self):
watchdog = Watchdog({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'required', 'safety_margin': -1}})
self.assertEqual(watchdog.activate(), False)
self.assertEqual(watchdog.is_running, False)
self.assertEquals(watchdog.activate(), False)
self.assertEquals(watchdog.is_running, False)
@patch('platform.system', Mock(return_value='Linux'))
@patch.object(LinuxWatchdogDevice, 'get_timeout', Mock(return_value=16))
def test_timeout_does_not_ensure_safe_termination(self):
Watchdog({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'auto', 'safety_margin': -1}}).activate()
self.assertEqual(len(mock_devices), 2)
self.assertEquals(len(mock_devices), 2)
@patch('platform.system', Mock(return_value='Linux'))
@patch.object(Watchdog, 'is_running', PropertyMock(return_value=False))
@@ -99,29 +99,29 @@ class TestWatchdog(unittest.TestCase):
watchdog = Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'required'}})
watchdog.activate()
self.assertEqual(len(mock_devices), 2)
self.assertEquals(len(mock_devices), 2)
device = mock_devices[-1]
self.assertTrue(device.open)
self.assertEqual(device.timeout, 24)
self.assertEquals(device.timeout, 24)
watchdog.keepalive()
self.assertEqual(len(device.writes), 1)
self.assertEquals(len(device.writes), 1)
watchdog.disable()
self.assertFalse(device.open)
self.assertEqual(device.writes[-1], b'V')
self.assertEquals(device.writes[-1], b'V')
def test_invalid_timings(self):
watchdog = Watchdog({'ttl': 30, 'loop_wait': 20, 'watchdog': {'mode': 'automatic', 'safety_margin': -1}})
watchdog.activate()
self.assertEqual(len(mock_devices), 1)
self.assertEquals(len(mock_devices), 1)
self.assertFalse(watchdog.is_running)
def test_parse_mode(self):
with patch('patroni.watchdog.base.logger.warning', new_callable=Mock()) as warning_mock:
watchdog = Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'bad'}})
self.assertEqual(watchdog.config.mode, 'off')
self.assertEquals(watchdog.config.mode, 'off')
warning_mock.assert_called_once()
@patch('platform.system', Mock(return_value='Unknown'))
@@ -132,11 +132,9 @@ class TestWatchdog(unittest.TestCase):
def test_exceptions(self):
wd = Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'bad'}})
wd.impl.close = wd.impl.keepalive = Mock(side_effect=WatchdogError(''))
self.assertTrue(wd.activate())
self.assertIsNone(wd.keepalive())
self.assertIsNone(wd.disable())
self.assertIsNone(wd.keepalive())
@patch('platform.system', Mock(return_value='Linux'))
def test_config_reload(self):
watchdog = Watchdog({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'required'}})
self.assertTrue(watchdog.activate())
@@ -170,7 +168,7 @@ class TestNullWatchdog(unittest.TestCase):
watchdog = NullWatchdog()
self.assertTrue(watchdog.can_be_disabled)
self.assertRaises(WatchdogError, watchdog.set_timeout, 1)
self.assertEqual(watchdog.describe(), 'NullWatchdog')
self.assertEquals(watchdog.describe(), 'NullWatchdog')
self.assertIsInstance(NullWatchdog.from_config({}), NullWatchdog)
@@ -194,24 +192,15 @@ class TestLinuxWatchdogDevice(unittest.TestCase):
self.assertRaises(WatchdogError, self.impl.set_timeout, -1)
@patch('os.open', Mock(return_value=3))
@patch('fcntl.ioctl', Mock(side_effect=OSError))
@patch('fcntl.ioctl', Mock(return_value=-1))
def test__ioctl(self):
self.assertRaises(WatchdogError, self.impl.get_support)
self.impl.open()
self.assertRaises(WatchdogError, self.impl.get_support)
self.assertRaises(IOError, self.impl.get_support)
def test_is_healthy(self):
self.assertFalse(self.impl.is_healthy)
@patch('os.open', Mock(return_value=3))
@patch('fcntl.ioctl', Mock(side_effect=OSError))
def test_error_handling(self):
self.impl.open()
self.assertRaises(WatchdogError, self.impl.get_timeout)
self.assertRaises(WatchdogError, self.impl.set_timeout, 10)
# We still try to output a reasonable string even if getting info errors
self.assertEqual(self.impl.describe(), "Linux watchdog device")
@patch('os.open', Mock(side_effect=OSError))
def test_open(self):
self.assertRaises(WatchdogError, self.impl.open)
+12 -22
View File
@@ -1,4 +1,3 @@
import select
import six
import unittest
@@ -59,11 +58,11 @@ class MockKazooClient(Mock):
raise TypeError("Invalid type for 'path' (string expected)")
if not isinstance(value, (six.binary_type,)):
raise TypeError("Invalid type for 'value' (must be a byte string)")
if b'Exception' in value:
if value == b'Exception':
raise Exception
if path.endswith('/initialize') or path == '/service/test/optime/leader':
raise Exception
elif b'retry' in value or (b'exists' in value and self.exists):
elif value == b'retry' or (value == b'exists' and self.exists):
raise NodeExistsError
def create_async(self, path, value=b"", acl=None, ephemeral=False, sequence=False, makepath=False):
@@ -77,10 +76,10 @@ class MockKazooClient(Mock):
raise TypeError("Invalid type for 'value' (must be a byte string)")
if path == '/service/bla/optime/leader':
raise Exception
if path == '/service/test/members/bar' and b'retry' in value:
if path == '/service/test/members/bar' and value == b'retry':
return
if path in ('/service/test/failover', '/service/test/config', '/service/test/sync'):
if b'Exception' in value:
if value == b'Exception':
raise Exception
elif value == b'ok':
return
@@ -114,11 +113,6 @@ class TestPatroniSequentialThreadingHandler(unittest.TestCase):
def test_create_connection(self):
self.assertIsNotNone(self.handler.create_connection(()))
self.assertIsNotNone(self.handler.create_connection((), 40))
self.assertIsNotNone(self.handler.create_connection(timeout=40))
@patch.object(SequentialThreadingHandler, 'select', Mock(side_effect=ValueError))
def test_select(self):
self.assertRaises(select.error, self.handler.select)
class TestZooKeeper(unittest.TestCase):
@@ -151,7 +145,7 @@ class TestZooKeeper(unittest.TestCase):
self.assertRaises(ZooKeeperError, self.zk.get_cluster)
cluster = self.zk.get_cluster()
self.assertIsInstance(cluster.leader, Leader)
self.zk.touch_member({'foo': 'foo'})
self.zk.touch_member('foo')
def test_delete_leader(self):
self.assertTrue(self.zk.delete_leader())
@@ -162,7 +156,7 @@ class TestZooKeeper(unittest.TestCase):
self.zk.set_failover_value('Exception')
def test_set_config_value(self):
self.zk.set_config_value('', 1)
self.zk.set_config_value('')
self.zk.set_config_value('ok')
self.zk.set_config_value('Exception')
@@ -175,18 +169,17 @@ class TestZooKeeper(unittest.TestCase):
def test_touch_member(self):
self.zk._name = 'buzz'
self.zk.get_cluster()
self.zk.touch_member({'new': 'new'})
self.zk.touch_member('new')
self.zk._name = 'bar'
self.zk.touch_member({'new': 'new'})
self.zk.touch_member('new')
self.zk._name = 'na'
self.zk._client.exists = 1
self.zk.touch_member({'Exception': 'Exception'})
self.zk.touch_member('Exception')
self.zk._name = 'bar'
self.zk.touch_member({'retry': 'retry'})
self.zk.touch_member('retry')
self.zk._fetch_cluster = True
self.zk.get_cluster()
self.zk.touch_member({'conn_url': 'postgres://repuser:rep-pass@localhost:5434/postgres',
'api_url': 'http://127.0.0.1:8009/patroni'})
self.zk.touch_member('retry')
def test_take_leader(self):
self.zk.take_leader()
@@ -194,7 +187,7 @@ class TestZooKeeper(unittest.TestCase):
self.zk.take_leader()
def test_update_leader(self):
self.assertTrue(self.zk.update_leader(None))
self.assertTrue(self.zk.update_leader())
def test_write_leader_optime(self):
self.zk.last_leader_operation = '0'
@@ -224,6 +217,3 @@ class TestZooKeeper(unittest.TestCase):
self.zk.set_sync_state_value('ok')
self.zk.set_sync_state_value('Exception')
self.zk.delete_sync_state()
def test_set_history_value(self):
self.zk.set_history_value('{}')