Release v3.0.2 (#2617)

- bump version
- update release notes
- update links to Postgres Slack
- simplify /sync health-check endpoint code
- update unit-tests to cover missing lines

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,48 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
-
----
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Environment**
--   Patroni version:
--   PostgreSQL version:
--   DCS (and its version):
-
-**Patroni configuration file**
-```
-Please copy&paste your Patroni configuration file here
-```
-
-**patronictl show-config**
-```
-Please copy&paste the output of "patronictl show-config" command here
-```
-
-**Have you checked Patroni logs?**
-Please provide a snippet of Patroni log files here
-
-**Have you checked PostgreSQL logs?**
-Please provide a snippet here
-
-**Have you tried to use GitHub issue search?**
-Maybe there is already a similar issue solved.
-
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,97 @@
+name: Bug Report
+description: Create a report to help us improve
+labels:
+- bug
+body:
+- type: markdown
+  attributes:
+    value: |
+      If you have a question please post it on channel [#patroni](https://postgresteam.slack.com/archives/C9XPYG92A) in the [PostgreSQL Slack](https://join.slack.com/t/postgresteam/shared_invite/zt-1qj14i9sj-E9WqIFlvcOiHsEk2yFEMjA).
+      Before reporting a bug please make sure to **reproduce it with the latest Patroni version**!
+      Please fill the form below and provide as much information as possible.
+      Not doing so may result in your bug not being addressed in a timely manner.
+
+- type: textarea
+  id: problem
+  attributes:
+    label: What happened?
+  validations:
+    required: true
+
+- type: textarea
+  id: repro
+  attributes:
+    label: How can we reproduce it (as minimally and precisely as possible)?
+  validations:
+    required: true
+
+- type: textarea
+  id: expected
+  attributes:
+    label: What did you expect to happen?
+  validations:
+    required: true
+
+- type: textarea
+  id: environment
+  attributes:
+    label: Patroni/PostgreSQL/DCS version
+    value: |
+      -   Patroni version: 
+      -   PostgreSQL version: 
+      -   DCS (and its version): 
+  validations:
+    required: true
+
+- type: textarea
+  id: patroniConfig
+  attributes:
+    label: Patroni configuration file
+    description: Please copy and paste Patroni configuration file here. This will be automatically formatted into code, so no need for backticks.
+    render: yaml
+  validations:
+    required: true
+
+- type: textarea
+  id: globalConfig
+  attributes:
+    label: patronictl show-config
+    description: Please copy and paste `patronictl show-config` output here. This will be automatically formatted into code, so no need for backticks.
+    render: yaml
+  validations:
+    required: true
+
+- type: textarea
+  id: patroniLogs
+  attributes:
+    label: Patroni log files
+    description: Please copy and paste any relevant Patroni log output. This will be automatically formatted into code, so no need for backticks.
+    render: shell
+  validations:
+    required: true
+
+- type: textarea
+  id: postgresLogs
+  attributes:
+    label: PostgreSQL log files
+    description: Please copy and paste any relevant PostgreSQL log output. This will be automatically formatted into code, so no need for backticks.
+    render: shell
+  validations:
+    required: true
+
+- type: checkboxes
+  id: issueSearch
+  attributes:
+    label: Have you tried to use GitHub issue search?
+    description: Maybe there is already a similar issue solved.
+    options:
+    - label: 'Yes'
+      required: true
+  validations:
+    required: true
+
+- type: textarea
+  id: additional
+  attributes:
+    label: Anything else we need to know?
+    description: Add any other context about the problem here.

.github/ISSUE_TEMPLATE/config.yml

@@ -1 +1,5 @@
 blank_issues_enabled: false
+contact_links:
+- name: Question
+  url: https://join.slack.com/t/postgresteam/shared_invite/zt-1qj14i9sj-E9WqIFlvcOiHsEk2yFEMjA
+  about: "Please ask questions on channel #patroni in the PostgreSQL Slack"

.github/workflows/install_deps.py

@@ -45,6 +45,8 @@ def install_packages(what):
     packages['exhibitor'] = packages['zookeeper']
     packages = packages.get(what, [])
     ver = versions.get(what)
+    if float(ver) >= 15:
+        packages += ['postgresql-{0}-citus-11.2'.format(ver)]
     subprocess.call(['sudo', 'apt-get', 'update', '-y'])
     return subprocess.call(['sudo', 'apt-get', 'install', '-y', 'postgresql-' + ver, 'expect-dev'] + packages)
 

.github/workflows/tests.yaml

@@ -110,12 +110,14 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - uses: nolar/setup-k3d-k3s@v1
       if: matrix.dcs == 'kubernetes'
-    - name: Add postgresql apt repo
+    - name: Add postgresql and citus apt repo
       run: |
         sudo apt-get update -y
-        sudo apt-get install -y wget ca-certificates gnupg
+        sudo apt-get install -y wget ca-certificates gnupg debian-archive-keyring apt-transport-https
         sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
         sudo sh -c 'wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg'
+        sudo sh -c 'echo "deb [signed-by=/etc/apt/trusted.gpg.d/citusdata_community.gpg] https://repos.citusdata.com/community/ubuntu/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/citusdata_community.list'
+        sudo sh -c 'wget -qO - https://repos.citusdata.com/community/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/citusdata_community.gpg'
       if: matrix.os == 'ubuntu'
     - name: Install dependencies
       run: python .github/workflows/install_deps.py

Dockerfile.citus

@@ -27,7 +27,7 @@ RUN set -ex \
                             python3-etcd python3-kazoo python3-pip busybox \
                             net-tools iputils-ping --fix-missing \
     && curl https://install.citusdata.com/community/deb.sh | bash \
-    && apt-get -y install postgresql-$PG_MAJOR-citus-11.1 \
+    && apt-get -y install postgresql-$PG_MAJOR-citus-11.2 \
     && pip3 install dumb-init \
 \
     # Cleanup all locales but en_US.UTF-8

README.rst

@@ -14,7 +14,7 @@ We call Patroni a "template" because it is far from being a one-size-fits-all or
 
 Currently supported PostgreSQL versions: 9.3 to 15.
 
-**Note to Citus users**: Starting from 3.0 Patroni nicely integrates with `Citus <https://www.citusdata.com>`__. Please check `Citus support <https://github.com/zalando/patroni/blob/master/docs/citus.rst>`__ page for more information.
+**Note to Citus users**: Starting from 3.0 Patroni nicely integrates with the `Citus <https://github.com/citusdata/citus>`__ database extension to Postgres. Please check the `Citus support page <https://github.com/zalando/patroni/blob/master/docs/citus.rst>`__ in the Patroni documentation for more info about how to use Patroni high availability together with a Citus distributed cluster.
 
 **Note to Kubernetes users**: Patroni can run natively on top of Kubernetes. Take a look at the `Kubernetes <https://github.com/zalando/patroni/blob/master/docs/kubernetes.rst>`__ chapter of the Patroni documentation.
 
@@ -49,7 +49,7 @@ We report new releases information `here <https://github.com/zalando/patroni/rel
 Community
 =========
 
-There are two places to connect with the Patroni community: `on github <https://github.com/zalando/patroni>`__, via Issues and PRs, and on channel `#patroni <https://postgresteam.slack.com/archives/C9XPYG92A>`__ in the `PostgreSQL Slack <https://postgresteam.slack.com/>`__.  If you're using Patroni, or just interested, please join us.
+There are two places to connect with the Patroni community: `on github <https://github.com/zalando/patroni>`__, via Issues and PRs, and on channel `#patroni <https://postgresteam.slack.com/archives/C9XPYG92A>`__ in the `PostgreSQL Slack <https://join.slack.com/t/postgresteam/shared_invite/zt-1qj14i9sj-E9WqIFlvcOiHsEk2yFEMjA>`__.  If you're using Patroni, or just interested, please join us.
 
 ===================================
 Technical Requirements/Installation

docs/CONTRIBUTING.rst

@@ -8,7 +8,7 @@ Wanna contribute to Patroni? Yay - here is how!
 Chatting
 --------
 
-Just want to chat with other Patroni users?  Looking for interactive troubleshooting help? Join us on channel `#patroni <https://postgresteam.slack.com/archives/C9XPYG92A>`__ in the `PostgreSQL Slack <https://postgresteam.slack.com/>`__.
+Just want to chat with other Patroni users?  Looking for interactive troubleshooting help? Join us on channel `#patroni <https://postgresteam.slack.com/archives/C9XPYG92A>`__ in the `PostgreSQL Slack <https://join.slack.com/t/postgresteam/shared_invite/zt-1qj14i9sj-E9WqIFlvcOiHsEk2yFEMjA>`__.
 
 Running tests
 -------------

docs/ENVIRONMENT.rst

@@ -117,6 +117,7 @@ Kubernetes
 -  **PATRONI\_KUBERNETES\_POD\_IP**: (optional) IP address of the pod Patroni is running in. This value is required when `PATRONI_KUBERNETES_USE_ENDPOINTS` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
 -  **PATRONI\_KUBERNETES\_PORTS**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``PATRONI_KUBERNETES_PORTS='[{"name": "postgresql", "port": 5432}]'`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `PATRONI_KUBERNETES_USE_ENDPOINTS` is set.
 -  **PATRONI\_KUBERNETES\_CACERT**: (optional) Specifies the file with the CA_BUNDLE file with certificates of trusted CAs to use while verifying Kubernetes API SSL certs. If not provided, patroni will use the value provided by the ServiceAccount secret.
+-  **PATRONI\_RETRIABLE\_HTTP\_CODES**: (optional) list of HTTP status codes from K8s API to retry on. By default Patroni is retrying on ``500``, ``503``, and ``504``, or if K8s API response has ``retry-after`` HTTP header.
 
 Raft (deprecated)
 -----------------

docs/SETTINGS.rst

@@ -218,6 +218,7 @@ Kubernetes
 -  **pod\_ip**: (optional) IP address of the pod Patroni is running in. This value is required when `use_endpoints` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
 -  **ports**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``kubernetes.ports: [{"name": "postgresql", "port": 5432}]`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `kubernetes.use_endpoints` is set.
 -  **cacert**: (optional) Specifies the file with the CA_BUNDLE file with certificates of trusted CAs to use while verifying Kubernetes API SSL certs. If not provided, patroni will use the value provided by the ServiceAccount secret.
+-  **retriable\_http\_codes**: (optional) list of HTTP status codes from K8s API to retry on. By default Patroni is retrying on ``500``, ``503``, and ``504``, or if K8s API response has ``retry-after`` HTTP header.
 
 
 .. _raft_settings:

docs/_static/multi-dc-asynchronous-replication.drawio

@@ -0,0 +1 @@
+<mxfile host="app.diagrams.net" modified="2023-03-13T14:29:21.924Z" agent="5.0 (X11; Ubuntu)" etag="sukwsRuBYbiX8e-LLYnw" version="21.0.6" type="device"><diagram name="Page-1" id="Xu3tU9JEMeQEUPilRV_D">7Vxtb9s2EP41BrYPNfTmt4+Jk2bFOixrihXYF4O2aEkNLaoUZTv99SMlUhZF+i2RE8dVEsDiiTxKd88deXeMO+54sb4jIAn/wj5EHcfy1x33puM4tud67INTngrKkLc4ISCRLzptCA/RTyiIlqBmkQ9TpSPFGNEoUYkzHMdwRhUaIASv1G5zjNRZExBAjfAwA0infot8Ggqq3R9tbvwBoyCk8v0GxY0FkJ3Fm6Qh8PGqQnJvO+6YYEyLq8V6DBEXnpRLMe7jlrvlgxEY00MG/L3474v99d/Hb+jnn596d99vsjj7ILgsAcrEC9+MhYJS+iSFkOAoprkge9fsj80ztjo9dmfMW12nVyPU2wOVYOstzkMl1NsDlWDX2du1+e36A1YIWkthb9XmtyoPyP7ca5xRFMVwXELOYsSAAD9iqhhjhAmjxThm0rsO6QKxls0uV2FE4UMCZlyqK2YujDbHMRWgtx3ZFoLnXBmsKWBzEcEj1wQkt0tYKKTogxBI0mhajiJwlpE0WsIvMC2YcyoDYMKvF+uA22oXrFKvGxCcJfnjf2JzGe9O2OVkhnDmcyaU4EcoX7LjuOz3Iwfc9TxCqPbyS0hoxGzpCkUB500xnwqIFoJzyjkyiURx8Dlv3biWkIJpCh+kIfTF6+j4l2Bms8J1hSTs4Q7iBaTkiXURd3vSNoVz8kRztbF0T9LCipF7chwQ3iUoWW8MkF0IGzzCHh3NHu8Bk3gcaTZpELemm95VfzzsVwVnb9VKHXk1HZSsTCiugFzXyk6/c7CqbHvAzXq3shyrpyur7Ni4slxdWXd8TJxSEDP5OH3EAT4l7CrIoc7o/vRJ02X6COksFII3epdtFrHF6xxmpYw+z3/qpiUR8hlMIbrHaUSj3DdMMaV4sdewZ5D7KBUX+xwdSJPibefRGvrbvBWBKc7IDBa+ivm51OS1/OlE6mAiRX5CZI5UJzLQcdk3+JD+qVDpHYtKAoHPOhCYIKbTFpyvB04u+YmU+wkR2lMRar81RHstRFuIKhB13DODaF+D6O3X8Q0PNFGWcu04lh4nKTisKE8BR76BSmthgIoqK/8x4bDEWz0k61pWHmR1+24t+BLxVY06MlKLOK3Wc7SF8SAfze4bmNg1mjOs9c0Dqb12olmE2XDqWH/MppDEkIm5GxVIT2R4wxTkn8zPDlUQu44O4qEpnBieCMQDDcQtZFvIViHbPzPEDjWAQj+AcqHDhIY4wDFAtxvqNcFZ7JdY3fT5jLmkczR/h5Q+ieUTZBS/JGYtVtAd/URmkAISwF38xBLDX3CnqghEgEZLNSFpkrwYes/trLK01r2S56ksigcVo2r6Kx/j+SodtU6odUI7nRDT23l5IVl8eNduaPBWbuhlorcvQPT9A0U/Oi/R6475ckU/OC/R66nkG74WFHkQtujFil76PzJeNSyWxA9iTbziYiQwF6nsIPMn7JMtgqPiChWUjwVb2aEt+bUlv03Jb4ZJggmgcOIDCiblPmJ7iel05T+9itVQ+c9Ttx1vX/2zDbn7yyz/ucfqyrbPrfpnt1nsS89iH49Sr16lfvNEtq0nAVuY/uIwdZzzg+lQg6lWcNFDwzZxdFGJo+P97blVXOw229mC9p3VXJyzznYK8e5N/JSnw/dlfuRKc+l1lzIebl1R64reS+XFgNF36IvkLuANfNHLpO9ehPSHB0pfyvFcpO/9UtKXRvL60n+kftr/Z/jtGi7DW39l/0juoeEwv8yM+NGyUkepJUsq9RRDv5xkKNzwhMKHIk/Pyzb9ZN3ZUrY5fjqVxGc65BFsd88zHMzIa4pRrylG+8R7MKNBU4yGTTEaNcSIeYCGGNlNMXKaYtQUsp1tyL6vWGXBTDPWX5qsuKV6BJIHDPJfax11bapvk+cIr2YhILTLq5JTkPIV0FSROtmG2altmAc9bb/slrnV6o6510Di1Lhu6QfVj1x87KYMbSsjY73hQLic2as0xigh0QJw9e+UhnHhf4aVMXRT1bT2BqqLyPeLHSY/UAA2Jw3UQJ5HxXxTKQ4d5Far5ABEkcdQr65UWdjW95RVuUGt2OHqUe7IFOWeymbPZKNvOFO1a2u8d0fvntWGXi/PS2MJ3WearjVDIE2VQTSiCL6Cw9l6BixntBKo5axiTBYAGZk9UALBIooD1u1LUWbME1iO9RtIn+JZSHCMs/T3ildRD4nt80FcsltcEEdFnjY7nR/SEb7L+jT/UX6JiJikU/2eDpNfsbqW5wwV1yJt4Lm5Y9kFz+cpPDItzJqbbxMpum++k8W9/R8=</diagram></mxfile>

docs/_static/multi-dc-synchronous-replication.drawio

@@ -0,0 +1 @@
+<mxfile host="app.diagrams.net" modified="2023-03-13T14:25:32.295Z" agent="5.0 (X11; Ubuntu)" etag="EcVEbU6F-AyuIGXoP3hl" version="21.0.6" type="device"><diagram id="SVgELWPNXIlR7V7eDs_m" name="Page-1">7Vtbc5s4FP41frQHgY3tx9hO2u5kZ9Km7c70xSODDGxlxAoRO/31K4FkgwS2k+DGTZxmpuggjsS5fOci0nGmq80HCpPwb+Ij3LEtf9NxZh3bBn2nz/8TlMeCMhyOCkJAI19O2hHuo19IEi1JzSIfpZWJjBDMoqRK9EgcI49VaJBSsq5OWxJcXTWBATII9x7EJvWfyGehpAJ3vLvxEUVBKJce2cPixgqqyfJN0hD6ZF0iOdcdZ0oJYcXVajNFWAhPyaV47qbh7nZjFMXsmAf8b7fdj9cAuzc/wOTz92/W3Y+vXafg8gBxJl94NrXlftmjEkJCopjlghxM+C9fZ2p1BvzOVIx69kAj6ONhlQDMkeBRJejjYZUAdPZAWx/oGywRjFGFvaWtb5U2yH+dCckYjmI03ZqcxYkBhX7EVTElmFBOi0nMpTcJ2QrzEeCX6zBi6D6BnpDqmrsLpy1JzKTRA1uNpeAFV27WDPK1qOSRawLR6wdUKKSYgzFM0mixfYoiL6Np9IC+oLRgLqjcABNxvdoEwld7cJ32ewElWZJv/xNfq/bunF/OPUwyXzBhlPxE6iU7tsP/3QiDmywjjLWXf0CURdyXrnAUCN6MiKWgHGG0ZIIjl0gUB7f5aOZYUgp1S/gwDZEvX8e0f+kSYlW0KZGkP3xAZIUYfeRT5N2+JX3zUY2L4Xrn6Y5y37Dk5I56Dkp0Cbasdw7IL6QPPsEf3b7hkHeQizyODKeskbehnMGVOx25ZcmBRrXopqcpYcuqzoxLVm6qZS/wHK0roOvKNnQF6nQFHPtEunKAoavrr9OZCEY4S7mX8itgqC39iZgXShmX5Fax7VzGqQYVJX13hAmKnzqlL/MfBRYl2O5ZVg7EPdfRAFpisEYd11ILLNdmjhsYD/On+f0aJkCj2SNtbg62ylhv4QLhO5JGLMpxakEYI6sSnHhIIGOjeevo9zNbIBojLuZelCPfJFEQyBXkn8yM7aoZ25aJOaMaM+6PXm7F6ch2HA96nxfjq1/hp+zT9C+vaxpxx3axAGY/euCXAcuduiAtqE7ha9bMy0klq3f/y0Sak2NKtwhJV3yCm2yKh+TtFy1XJYmVjtkCcA7s4WhG/bYYDdpidEi8RzMatsVo1BajcUuMuIu3xAi0xchui1Fblm03WfZdySsLZoazvmtyBZb0NCCP2qqktKu5gB6rlpisvRBS1vMhgwuYooY8rCEaHRvImqNWvxq1RMWlJ8rD3sAMW4MWEuXasGXvD1tHQEhbftbI6O6DeIk4ZTDmOnqatZzZq7TGKKHRCgr175VGbdx/hpNxL2BVzzqYLK4i3xeP8xqavw7c1dTVZFpkpjBjRJbXudNW8nBZkdUUaaWS3+6f0mfBoOKzwDEzzXFdpnmq2tYsbWeiBig0yYuduNaycpjrylpI2FZCUZON8uJnXLWOgm2DeVzaWu+6reURmhAKGZqLqDrfQkJzF+V0LS6zUdNai+vMOlxgbHa46gLlLgZQBHk5blGUYC7p/Q2VWhtuaic22PZxLc5yo6WitIMNi/0mszXDkosdcieYJsXbLqMN8pt8gkc0klEPFR4hAlydb/iLuVLFXEh+ruR+dGuv/1Qb3UYmaaSu2dpza2zUPZWJqu1cGnvvt7H3dCN+xcZe/VGCWSK9zaOEp6vq8LHPbz5KAGYXdjZ1LgnrJWF9F+ewQ/fsstThJUt921kqeLKVnl2a6hpWhvwAKZETykISkBji6x11woEk9rdWsJtzS4R+cz3+ixh7lIoUPa5nBWUl3kKZ+95CSlFsfa8WKMKQcSytYEydUOWjdwKid9rrAg1kOMxUeTBIA8TkY5putvt4gbrMfOxSVbyzquIZoHN2ZYX7TsqKZ+jq7D5RskfnESLUl7s5wu6T+fj3BIOBFsoNDRRh63SxwEwvVUYZOnlEgGmxE3XKwSKGUQsHc219pnDwqLDxQCdntJbmL1jFhK4grmV2/xh7IccWkqV84pcix8sRvemI55lH9ULqDadEwhbzaNI52UnikYikvLnmrFB+/i5X6ZS/MK9Dqi4P7mPHqfiA+hLsua6lppDlMkUn8Rp7/Ieh2fA3odn4ldFM2WUDmh0GE/MrrBNCYFuffLXH6ALKLZD/DAhXp58vhnCO4CN3WEVw59wR3DF72q+J4IebE9aRUK/+FqA9qH9Za6j/h8n52JCqFHIuch68VTm33pVrkDMf7v4EsoCZ3R+SOtf/Aw==</diagram></mxfile>

docs/citus.rst

@@ -12,9 +12,10 @@ TL;DR
 
 There are only a few simple rules you need to follow:
 
-1. Citus extension must be available on all nodes. Absolute minimum supported
+1. `Citus <https://github.com/citusdata/citus>`__ database extension to
-   Citus version is 10.0, but, to take all benefits from transparent
+   PostgreSQL must be available on all nodes.  Absolute minimum supported Citus
-   switchovers and restarts of workers we recommend using at least Citus 11.2.
+   version is 10.0, but, to take all benefits from transparent switchovers and
+   restarts of workers we recommend using at least Citus 11.2.
 2. Cluster name (``scope``) must be the same for all Citus nodes!
 3. Superuser credentials must be the same on coordinator and all worker
    nodes, and ``pg_hba.conf`` should allow superuser access between all nodes.
@@ -51,8 +52,10 @@ patronictl
 ----------
 
 Coordinator and worker clusters are physically different PostgreSQL/Patroni
-clusters that are just logically groupped together using Citus. Therefore in
+clusters that are just logically groupped together using the
-most cases it is not possible to manage them as a single entity.
+`Citus <https://github.com/citusdata/citus>`__ database extension to
+PostgreSQL. Therefore in most cases it is not possible to manage them as a
+single entity.
 
 It results in two major differences in ``patronictl`` behaviour when
 ``patroni.yaml`` has the ``citus`` section comparing with the usual:

docs/ha_multi_dc.rst

@@ -0,0 +1,54 @@
+.. _ha_multi_dc:
+
+=================
+HA multi datacenter
+=================
+
+The high availability of a PostgreSQL cluster deployed in multiple data centers is based on replication, which can be synchronous or asynchronous (`replication_modes <replication_modes.rst>`_).
+
+In both cases, it is important to be clear about the following concepts:
+
+- Postgres can run as primary or standby leader only when it owns the leading key and can update the leading key.
+- You should run the odd number of etcd, ZooKeeper or Consul nodes: 3 or 5!
+
+Synchronous Replication
+----------------------------
+
+To have a multi DC cluster that can automatically tolerate a zone drop, a minimum of 3 is required.
+
+The architecture diagram would be the following:
+
+.. image:: _static/multi-dc-synchronous-replication.png
+
+We must deploy a cluster of etcd, ZooKeeper or Consul through the different DC, with a minimum of 3 nodes, one in each zone.
+
+Regarding postgres, we must deploy at least 2 nodes, in different DC. Then you have to set ``synchronous_mode: true`` in the global configuration (``patronictl edit-config``).
+
+This enables sync replication and the primary node will choose one of the nodes as synchronous.
+
+Asynchronous Replication
+----------------------------------
+
+With only two data centers it would be better to have two independent etcd clusters and run Patroni :ref:`standby cluster <standby_cluster>` in the second data center. If the first site is down, you can MANUALLY promote the ``standby_cluster``.
+
+The architecture diagram would be the following:
+
+.. image:: _static/multi-dc-asynchronous-replication.png
+
+Automatic promotion is not possible, because DC2 will never able to figure out the state of DC1.
+
+You should not use ``pg_ctl promote`` in this scenario, you need "manually promote" the healthy cluster with ``patronictl edit-config`` and remove ``standby_cluster`` section from there.
+
+.. warning::
+    If the source cluster is still up and running and you promote the standby cluster you create a split-brain.
+
+In case you want to return to the "initial" state, there are only two ways of resolving it:
+
+- Add the standby_cluster section back and it will trigger pg_rewind, but there are chances that pg_rewind will fail.
+- Rebuild the standby cluster from scratch.
+
+Before promoting standby cluster one have to manually ensure that the source cluster is down (STONITH). When DC1 recovers, the cluster has to be converted to a standby cluster.
+
+Before doing that you may manually examine the database and extract all changes that happened between the time when network between DC1 and DC2 has stopped working and the time when you manually stopped the cluster in DC1.
+
+Once extracted, you may also manually apply these changes to the cluster in DC2.

docs/index.rst

@@ -12,7 +12,7 @@ We call Patroni a "template" because it is far from being a one-size-fits-all or
 
 Currently supported PostgreSQL versions: 9.3 to 15.
 
-**Note to Citus users**: Starting from 3.0 Patroni nicely integrates with `Citus <https://www.citusdata.com>`__. Please check :ref:`Citus support <citus>` page for more information.
+**Note to Citus users**: Starting from 3.0 Patroni nicely integrates with the `Citus <https://github.com/citusdata/citus>`__ database extension to Postgres. Please check the :ref:`Citus support page <citus>` in the Patroni documentation for more info about how to use Patroni high availability together with a Citus distributed cluster.
 
 **Note to Kubernetes users**: Patroni can run natively on top of Kubernetes. Take a look at the :ref:`Kubernetes <kubernetes>` chapter of the Patroni documentation.
 
@@ -32,6 +32,7 @@ Currently supported PostgreSQL versions: 9.3 to 15.
    security
    replica_bootstrap
    replication_modes
+   ha_multi_dc
    pause
    kubernetes
    watchdog

docs/releases.rst

@@ -3,6 +3,72 @@
 Release notes
 =============
 
+Version 3.0.2
+-------------
+
+.. warning::
+    Version 3.0.2 dropped support of Python older than 3.6.
+
+
+**New features**
+
+- Added sync standby replica status to ``/metrics`` endpoint (Thomas von Dein, Alexander Kukushkin)
+
+  Before were only reporting ``primary``/``standby_leader``/``replica``.
+
+- User-friendly handling of ``PAGER`` in ``patronictl`` (Israel Barth Rubio)
+
+  It makes pager configurable via ``PAGER`` environment variable, which overrides default ``less`` and ``more``.
+
+- Make K8s retriable HTTP status code configurable (Alexander)
+
+  On some managed platforms it is possible to get status code ``401 Unauthorized``, which sometimes gets resolved after a few retries.
+
+
+**Improvements**
+
+- Set ``hot_standby`` to ``off`` during custom bootstrap only if ``recovery_target_action`` is set to ``promote`` (Alexander)
+
+  It was necessary to make ``recovery_target_action=pause`` work correctly.
+
+- Don't allow ``on_reload`` callback to kill other callbacks (Alexander)
+
+  ``on_start``/``on_stop``/``on_role_change`` are usually used to add/remove Virtual IP and ``on_reload`` should not interfere with them.
+
+- Switched to ``IMDSFetcher`` in aws callback example script (Polina Bungina)
+
+  The ``IMDSv2`` requires a token to work with and the ``IMDSFetcher`` handles it transparently.
+
+
+**Bugfixes**
+
+- Fixed ``patronictl switchover`` on Citus cluster running on Kubernetes (Lukáš Lalinský)
+
+  It didn't work for namespaces different from ``default``.
+
+- Don't write to ``PGDATA`` if major version is not known (Alexander)
+
+  If right after the start ``PGDATA`` was empty (maybe wasn't yet mounted), Patroni was making a false assumption about PostgreSQL version and falsely creating ``recovery.conf`` file even if the actual major version is v10+.
+
+- Fixed bug with Citus metadata after coordinator failover (Alexander)
+
+  The ``citus_set_coordinator_host()`` call doesn't cause metadata sync and the change was invisible on worker nodes. The issue is solved by switching to ``citus_update_node()``.
+
+- Use etcd hosts listed in the config file as a fallback when all etcd nodes "failed" (Alexander)
+
+  The etcd cluster may change topology over time and Patroni tries to follow it. If at some point all nodes became unreachable Patroni will use a combination of nodes from the config plus the last known topology when trying to reconnect.
+
+
+Version 3.0.1
+-------------
+
+**Bugfixes**
+
+- Pass proper role name to an ``on_role_change`` callback script'. (Alexander Kukushkin, Polina Bungina)
+
+  Patroni used to erroneously pass ``promoted`` role to an ``on_role_change`` callback script on promotion. The passed role name changed back to ``master``. This regression was introduced in 3.0.0.
+
+
 Version 3.0.0
 -------------
 

docs/rest_api.rst

@@ -111,6 +111,69 @@ The ``GET /patroni`` is used by Patroni during the leader race. It also could be
       }
     }
 
+
+Retrieve the Patroni metrics in Prometheus format through the ``GET /metrics`` endpoint.
+
+.. code-block:: bash
+
+	$ curl http://localhost:8008/metrics
+	
+	# HELP patroni_version Patroni semver without periods. \
+	# TYPE patroni_version gauge
+	patroni_version{scope="batman"} 020103
+	# HELP patroni_postgres_running Value is 1 if Postgres is running, 0 otherwise.
+	# TYPE patroni_postgres_running gauge
+	patroni_postgres_running{scope="batman"} 1
+	# HELP patroni_postmaster_start_time Epoch seconds since Postgres started.
+	# TYPE patroni_postmaster_start_time gauge
+	patroni_postmaster_start_time{scope="batman"} 1657656955.179243
+	# HELP patroni_master Value is 1 if this node is the leader, 0 otherwise.
+	# TYPE patroni_master gauge
+	patroni_master{scope="batman"} 1
+	# HELP patroni_xlog_location Current location of the Postgres transaction log, 0 if this node is not the leader.
+	# TYPE patroni_xlog_location counter
+	patroni_xlog_location{scope="batman"} 22320573386952
+	# HELP patroni_standby_leader Value is 1 if this node is the standby_leader, 0 otherwise.
+	# TYPE patroni_standby_leader gauge
+	patroni_standby_leader{scope="batman"} 0
+	# HELP patroni_replica Value is 1 if this node is a replica, 0 otherwise.
+	# TYPE patroni_replica gauge
+	patroni_replica{scope="batman"} 0
+	# HELP patroni_sync_standby Value is 1 if this node is a sync standby replica, 0 otherwise.
+        # TYPE patroni_sync_standby gauge
+        patroni_sync_standby{scope="batman"} 0
+	# HELP patroni_xlog_received_location Current location of the received Postgres transaction log, 0 if this node is not a replica.
+	# TYPE patroni_xlog_received_location counter
+	patroni_xlog_received_location{scope="batman"} 0
+	# HELP patroni_xlog_replayed_location Current location of the replayed Postgres transaction log, 0 if this node is not a replica.
+	# TYPE patroni_xlog_replayed_location counter
+	patroni_xlog_replayed_location{scope="batman"} 0
+	# HELP patroni_xlog_replayed_timestamp Current timestamp of the replayed Postgres transaction log, 0 if null.
+	# TYPE patroni_xlog_replayed_timestamp gauge
+	patroni_xlog_replayed_timestamp{scope="batman"} 0
+	# HELP patroni_xlog_paused Value is 1 if the Postgres xlog is paused, 0 otherwise.
+	# TYPE patroni_xlog_paused gauge
+	patroni_xlog_paused{scope="batman"} 0
+	# HELP patroni_postgres_server_version Version of Postgres (if running), 0 otherwise.
+	# TYPE patroni_postgres_server_version gauge
+	patroni_postgres_server_version {scope="batman"} 140004
+	# HELP patroni_cluster_unlocked Value is 1 if the cluster is unlocked, 0 if locked.
+	# TYPE patroni_cluster_unlocked gauge
+	patroni_cluster_unlocked{scope="batman"} 0
+	# HELP patroni_postgres_timeline Postgres timeline of this node (if running), 0 otherwise.
+	# TYPE patroni_postgres_timeline counter
+	patroni_postgres_timeline{scope="batman"} 24
+	# HELP patroni_dcs_last_seen Epoch timestamp when DCS was last contacted successfully by Patroni.
+	# TYPE patroni_dcs_last_seen gauge
+	patroni_dcs_last_seen{scope="batman"} 1677658321
+	# HELP patroni_pending_restart Value is 1 if the node needs a restart, 0 otherwise.
+	# TYPE patroni_pending_restart gauge
+	patroni_pending_restart{scope="batman"} 1
+	# HELP patroni_is_paused Value is 1 if auto failover is disabled, 0 otherwise.
+	# TYPE patroni_is_paused gauge
+	patroni_is_paused{scope="batman"} 1
+
+
 Cluster status endpoints
 ------------------------
 

features/basic_replication.feature

@@ -65,6 +65,7 @@ Feature: basic replication
     Then I receive a response returncode 0
     And postgres2 role is the primary after 24 seconds
     And Response on GET http://127.0.0.1:8010/history contains recovery after 10 seconds
+    And there is a postgres2_cb.log with "on_role_change master batman" in postgres2 data directory
     When I issue a PATCH request to http://127.0.0.1:8010/config with {"synchronous_mode": null, "master_start_timeout": 0}
     Then I receive a response code 200
     When I add the table bar to postgres2

features/citus.feature

@@ -10,20 +10,20 @@ Feature: citus
     And I start postgres3 in citus group 1
     Then replication works from postgres0 to postgres1 after 15 seconds
     Then replication works from postgres2 to postgres3 after 15 seconds
-    And postgres0 is registered in the coordinator postgres0 as the worker in group 0
+    And postgres0 is registered in the postgres0 as the worker in group 0
-    And postgres2 is registered in the coordinator postgres0 as the worker in group 1
+    And postgres2 is registered in the postgres0 as the worker in group 1
 
   Scenario: coordinator failover updates pg_dist_node
     Given I run patronictl.py failover batman --group 0 --candidate postgres1 --force
     Then postgres1 role is the primary after 10 seconds
     And replication works from postgres1 to postgres0 after 15 seconds
     And "sync" key in a group 0 in DCS has sync_standby=postgres0 after 15 seconds
-    And postgres1 is registered in the coordinator postgres1 as the worker in group 0
+    And postgres1 is registered in the postgres2 as the worker in group 0
     When I run patronictl.py failover batman --group 0 --candidate postgres0 --force
     Then postgres0 role is the primary after 10 seconds
     And replication works from postgres0 to postgres1 after 15 seconds
     And "sync" key in a group 0 in DCS has sync_standby=postgres1 after 15 seconds
-    And postgres0 is registered in the coordinator postgres0 as the worker in group 0
+    And postgres0 is registered in the postgres2 as the worker in group 0
 
   Scenario: worker switchover doesn't break client queries on the coordinator
     Given I create a distributed table on postgres0
@@ -33,14 +33,14 @@ Feature: citus
     And postgres3 role is the primary after 10 seconds
     And replication works from postgres3 to postgres2 after 15 seconds
     And "sync" key in a group 1 in DCS has sync_standby=postgres2 after 15 seconds
-    And postgres3 is registered in the coordinator postgres0 as the worker in group 1
+    And postgres3 is registered in the postgres0 as the worker in group 1
     And a thread is still alive
     When I run patronictl.py switchover batman --group 1 --force
     Then I receive a response returncode 0
     And postgres2 role is the primary after 10 seconds
     And replication works from postgres2 to postgres3 after 15 seconds
     And "sync" key in a group 1 in DCS has sync_standby=postgres3 after 15 seconds
-    And postgres2 is registered in the coordinator postgres0 as the worker in group 1
+    And postgres2 is registered in the postgres0 as the worker in group 1
     And a thread is still alive
     When I stop a thread
     Then a distributed table on postgres0 has expected rows
@@ -52,7 +52,7 @@ Feature: citus
     Then I receive a response returncode 0
     And postgres2 role is the primary after 10 seconds
     And replication works from postgres2 to postgres3 after 15 seconds
-    And postgres2 is registered in the coordinator postgres0 as the worker in group 1
+    And postgres2 is registered in the postgres0 as the worker in group 1
     And a thread is still alive
     When I stop a thread
     Then a distributed table on postgres0 has expected rows
@@ -65,7 +65,7 @@ Feature: citus
     Then I receive a response returncode 0
     And I receive a response output "+ttl: 20"
     When I sleep for 2 seconds
-    Then postgres4 is registered in the coordinator postgres0 as the worker in group 2
+    Then postgres4 is registered in the postgres2 as the worker in group 2
     When I shut down postgres4
     Then There is a transaction in progress on postgres0 changing pg_dist_node
     When I run patronictl.py restart batman postgres2 --group 1 --force

features/environment.py

@@ -7,7 +7,6 @@ import psutil
 import re
 import shutil
 import signal
-import six
 import subprocess
 import sys
 import tempfile
@@ -17,12 +16,11 @@ import yaml
 
 import patroni.psycopg as psycopg
 
+from http.server import BaseHTTPRequestHandler, HTTPServer
 from patroni.request import PatroniRequest
-from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 
 
-@six.add_metaclass(abc.ABCMeta)
+class AbstractController(abc.ABC):
-class AbstractController(object):
 
     def __init__(self, context, name, work_directory, output_dir):
         self._context = context
@@ -201,6 +199,8 @@ class PatroniController(AbstractController):
         config['name'] = name
         config['postgresql']['data_dir'] = self._data_dir.replace('\\', '/')
         config['postgresql']['basebackup'] = [{'checkpoint': 'fast'}]
+        config['postgresql']['callbacks'] = {
+            'on_role_change': '{0} features/callback2.py {1}'.format(self._context.pctl.PYTHON, name)}
         config['postgresql']['use_unix_socket'] = os.name != 'nt'  # windows doesn't yet support unix-domain sockets
         config['postgresql']['use_unix_socket_repl'] = os.name != 'nt'
         config['postgresql']['pgpass'] = os.path.join(tempfile.gettempdir(), 'pgpass_' + name).replace('\\', '/')

features/steps/cascading_replication.py

@@ -11,11 +11,8 @@ def start_patroni_with_a_name_value_tag(context, name, tag_name, tag_value):
 
 @then('There is a {label} with "{content}" in {name:w} data directory')
 def check_label(context, label, content, name):
-    label = context.pctl.read_label(name, label)
+    value = (context.pctl.read_label(name, label) or '').replace('\n', '\\n')
-    if label is None:
+    assert content in value, "\"{0}\" in {1} doesn't contain {2}".format(value, label, content)
-        label = ""
-    label = label.replace('\n', '\\n')
-    assert content in label, "\"{0}\" doesn't contain {1}".format(label, content)
 
 
 @step('I create label with "{content:w}" in {name:w} data directory')

features/steps/citus.py

@@ -44,7 +44,7 @@ def start_citus(context, name, group):
     return context.pctl.start(name, custom_config={"citus": {"database": "postgres", "group": int(group)}})
 
 
-@step('{name1:w} is registered in the coordinator {name2:w} as the worker in group {group:d}')
+@step('{name1:w} is registered in the {name2:w} as the worker in group {group:d}')
 def check_registration(context, name1, name2, group):
     worker_port = int(context.pctl.query(name1, "SHOW port").fetchone()[0])
     r = context.pctl.query(name2, "SELECT nodeport FROM pg_catalog.pg_dist_node WHERE groupid = {0}".format(group))

kubernetes/Dockerfile.citus

@@ -11,7 +11,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \
     ## Make sure we have a en_US.UTF-8 locale available
     && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
     && curl https://install.citusdata.com/community/deb.sh | bash \
-    && apt-get -y install postgresql-15-citus-11.1 \
+    && apt-get -y install postgresql-15-citus-11.2 \
     && pip3 install setuptools \
     && pip3 install 'git+https://github.com/zalando/patroni.git#egg=patroni[kubernetes]' \
     && PGHOME=/home/postgres \

patroni/api.py

@@ -7,15 +7,14 @@ import traceback
 import dateutil.parser
 import datetime
 import os
-import six
 import socket
 import sys
 
-from ipaddress import ip_address, ip_network as _ip_network
+from http.server import BaseHTTPRequestHandler, HTTPServer
-from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+from ipaddress import ip_address, ip_network
-from six.moves.socketserver import ThreadingMixIn
+from socketserver import ThreadingMixIn
-from six.moves.urllib_parse import urlparse, parse_qs
 from threading import Thread
+from urllib.parse import urlparse, parse_qs
 
 from . import psycopg
 from .exceptions import PostgresConnectionException, PostgresException
@@ -26,10 +25,6 @@ from .utils import deep_compare, enable_keepalive, parse_bool, patch_config, Ret
 logger = logging.getLogger(__name__)
 
 
-def ip_network(value):
-    return _ip_network(value.decode('utf-8') if six.PY2 else value, False)
-
-
 class RestApiHandler(BaseHTTPRequestHandler):
 
     def _write_status_code_only(self, status_code):
@@ -147,8 +142,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         elif 'health' in path:
             status_code = 200 if response.get('state') == 'running' else 503
         elif cluster:  # dcs is available
-            is_synchronous = cluster.is_synchronous_mode() and cluster.sync \
+            is_synchronous = response.get('sync_standby')
-                    and patroni.postgresql.name in cluster.sync.members
             if path in ('/sync', '/synchronous') and is_synchronous:
                 status_code = replica_status_code
             elif path in ('/async', '/asynchronous') and not is_synchronous:
@@ -172,7 +166,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 if instance_tag_value is None:
                     status_code = 503
                     break
-                if not isinstance(instance_tag_value, six.string_types):
+                if not isinstance(instance_tag_value, str):
                     instance_tag_value = str(instance_tag_value).lower()
                 if instance_tag_value != qs_value:
                     status_code = 503
@@ -274,6 +268,10 @@ class RestApiHandler(BaseHTTPRequestHandler):
         metrics.append("# TYPE patroni_replica gauge")
         metrics.append("patroni_replica{0} {1}".format(scope_label, int(postgres['role'] == 'replica')))
 
+        metrics.append("# HELP patroni_sync_standby Value is 1 if this node is a sync standby replica, 0 otherwise.")
+        metrics.append("# TYPE patroni_sync_standby gauge")
+        metrics.append("patroni_sync_standby{0} {1}".format(scope_label, int(postgres.get('sync_standby', False))))
+
         metrics.append("# HELP patroni_xlog_received_location Current location of the received"
                        " Postgres transaction log, 0 if this node is not a replica.")
         metrics.append("# TYPE patroni_xlog_received_location counter")
@@ -691,6 +689,10 @@ class RestApiHandler(BaseHTTPRequestHandler):
             if result['role'] == 'replica' and self.server.patroni.ha.is_standby_cluster():
                 result['role'] = postgresql.role
 
+            if result['role'] == 'replica' and cluster and cluster.is_synchronous_mode()\
+                    and cluster.sync and postgresql.name in cluster.sync.members:
+                result['sync_standby'] = True
+
             if row[1] > 0:
                 result['timeline'] = row[1]
             else:
@@ -768,7 +770,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
     def __resolve_ips(host, port):
         try:
             for _, _, _, _, sa in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP):
-                yield ip_network(sa[0])
+                yield ip_network(sa[0], False)
         except Exception as e:
             logger.error('Failed to resolve %s: %r', host, e)
 
@@ -789,8 +791,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
 
     def check_access(self, rh):
         if self.__allowlist or self.__allowlist_include_members:
-            incoming_ip = rh.client_address[0]
+            incoming_ip = ip_address(rh.client_address[0])
-            incoming_ip = ip_address(incoming_ip.decode('utf-8') if six.PY2 else incoming_ip)
             if not any(incoming_ip in net for net in self.__allowlist + tuple(self.__members_ips())):
                 return rh._write_response(403, 'Access is denied')
 
@@ -915,7 +916,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
             for v in value:
                 if '/' in v:  # netmask
                     try:
-                        yield ip_network(v)
+                        yield ip_network(v, False)
                     except Exception as e:
                         logger.error('Invalid value "%s" in the allowlist: %r', v, e)
                 else:  # ip or hostname, try to resolve it
@@ -935,7 +936,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         self.http_extra_headers = config.get('http_extra_headers') or {}
         self.http_extra_headers.update((config.get('https_extra_headers') or {}) if ssl_options.get('certfile') else {})
 
-        if isinstance(config.get('verify_client'), six.string_types):
+        if isinstance(config.get('verify_client'), str):
             ssl_options['verify_client'] = config['verify_client'].lower()
 
         if self.__listen != config['listen'] or self.__ssl_options != ssl_options or self._received_new_cert:

patroni/config.py

@@ -2,7 +2,6 @@ import json
 import logging
 import os
 import shutil
-import six
 import tempfile
 import yaml
 
@@ -363,8 +362,8 @@ class Config(object):
                               'CACERT', 'CERT', 'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC', 'CONSISTENCY',
                               'REGISTER_SERVICE', 'SERVICE_CHECK_INTERVAL', 'SERVICE_CHECK_TLS_SERVER_NAME',
                               'SERVICE_TAGS', 'NAMESPACE', 'CONTEXT', 'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL',
-                              'POD_IP', 'PORTS', 'LABELS', 'BYPASS_API_SERVICE', 'KEY_PASSWORD', 'USE_SSL', 'SET_ACLS',
+                              'POD_IP', 'PORTS', 'LABELS', 'BYPASS_API_SERVICE', 'RETRIABLE_HTTP_CODES', 'KEY_PASSWORD',
-                              'GROUP', 'DATABASE') and name:
+                              'USE_SSL', 'SET_ACLS', 'GROUP', 'DATABASE') and name:
                     value = os.environ.pop(param)
                     if name == 'CITUS':
                         if suffix == 'GROUP':
@@ -373,7 +372,7 @@ class Config(object):
                             continue
                     elif suffix == 'PORT':
                         value = value and parse_int(value)
-                    elif suffix in ('HOSTS', 'PORTS', 'CHECKS', 'SERVICE_TAGS'):
+                    elif suffix in ('HOSTS', 'PORTS', 'CHECKS', 'SERVICE_TAGS', 'RETRIABLE_HTTP_CODES'):
                         value = value and _parse_list(value)
                     elif suffix in ('LABELS', 'SET_ACLS'):
                         value = _parse_dict(value)
@@ -408,8 +407,8 @@ class Config(object):
         config = self._safe_copy_dynamic_configuration(dynamic_configuration)
         for name, value in local_configuration.items():
             if name == 'citus':  # remove invalid citus configuration
-                if isinstance(value, dict) and isinstance(value.get('group'), six.integer_types)\
+                if isinstance(value, dict) and isinstance(value.get('group'), int)\
-                        and isinstance(value.get('database'), six.string_types):
+                        and isinstance(value.get('database'), str):
                     config[name] = value
             elif name == 'postgresql':
                 for name, value in (value or {}).items():

patroni/ctl.py

@@ -4,17 +4,17 @@ Patroni Control
 
 import click
 import codecs
+import copy
 import datetime
 import dateutil.parser
 import dateutil.tz
-import copy
 import difflib
 import io
 import json
 import logging
 import os
 import random
-import six
+import shutil
 import subprocess
 import sys
 import tempfile
@@ -25,7 +25,7 @@ from click import ClickException
 from collections import defaultdict
 from contextlib import contextmanager
 from prettytable import ALL, FRAME, PrettyTable
-from six.moves.urllib_parse import urlparse
+from urllib.parse import urlparse
 
 try:
     from ydiff import markup_to_pager, PatchStream
@@ -35,7 +35,7 @@ except ImportError:  # pragma: no cover
 from .dcs import get_dcs as _get_dcs
 from .exceptions import PatroniException
 from .postgresql.misc import postgres_version_to_int
-from .utils import cluster_as_json, find_executable, patch_config, polling_loop, is_standby_cluster
+from .utils import cluster_as_json, patch_config, polling_loop, is_standby_cluster
 from .request import PatroniRequest
 from .version import __version__
 
@@ -203,7 +203,7 @@ def print_output(columns, rows, alignment=None, fmt='pretty', header=None, delim
             for r in ([columns] if columns else []) + rows:
                 click.echo(delimiter.join(map(str, r)))
         else:
-            hrules = ALL if any(any(isinstance(c, six.string_types) and '\n' in c for c in r) for r in rows) else FRAME
+            hrules = ALL if any(any(isinstance(c, str) and '\n' in c for c in r) for r in rows) else FRAME
             table = PatronictlPrettyTable(header, columns, hrules=hrules)
             table.align = 'l'
             for k, v in (alignment or {}).items():
@@ -875,7 +875,7 @@ def output_members(obj, cluster, name, extended=False, fmt='pretty', group=None)
             member.update(cluster=name, member=member['name'], group=g,
                           host=member.get('host', ''), tl=member.get('timeline', ''),
                           role=member['role'].replace('_', ' ').title(),
-                          lag_in_mb=round(lag/1024/1024) if isinstance(lag, six.integer_types) else lag,
+                          lag_in_mb=round(lag/1024/1024) if isinstance(lag, int) else lag,
                           pending_restart='*' if member.get('pending_restart') else '')
 
             if append_port and member['host'] and member.get('port'):
@@ -1084,8 +1084,7 @@ def show_diff(before_editing, after_editing):
     if sys.stdout.isatty():
         buf = io.StringIO()
         for line in unified_diff:
-            # Force cast to unicode as difflib on Python 2.7 returns a mix of unicode and str.
+            buf.write(str(line))
-            buf.write(six.text_type(line))
         buf.seek(0)
 
         class opts:
@@ -1093,12 +1092,28 @@ def show_diff(before_editing, after_editing):
             width = 80
             tab_width = 8
             wrap = True
-            if find_executable('less'):
+            pager = next(
-                pager = None
+                (
-            else:
+                    os.path.basename(p)
-                pager = 'more.com' if sys.platform == 'win32' else 'more'
+                    for p in (os.environ.get('PAGER'), "less", "more")
+                    if p is not None and shutil.which(p)
+                ),
+                None,
+            )
             pager_options = None
 
+        if opts.pager is None:
+            raise PatroniCtlException(
+                'No pager could be found. Either set PAGER environment variable with '
+                'your pager or install either "less" or "more" in the host.'
+            )
+
+        # if we end up selecting "less" as "pager" then we set "pager" attribute
+        # to "None". "less" is the default pager for "ydiff" module, and that
+        # module adds some command-line options to "less" when "pager" is "None"
+        if opts.pager == 'less':
+            opts.pager = None
+
         markup_to_pager(PatchStream(buf), opts)
     else:
         for line in unified_diff:
@@ -1184,7 +1199,7 @@ def invoke_editor(before_editing, cluster_name):
     editor_cmd = os.environ.get('EDITOR')
     if not editor_cmd:
         for editor in ('editor', 'vi'):
-            editor_cmd = find_executable(editor)
+            editor_cmd = shutil.which(editor)
             if editor_cmd:
                 logging.debug('Setting fallback editor_cmd=%s', editor)
                 break

patroni/daemon.py

@@ -1,18 +1,40 @@
+"""Daemon processes abstraction module.
+
+This module implements abstraction classes and functions for creating and managing daemon processes in Patroni.
+Currently it is only used for the main "Thread" of ``patroni`` and ``patroni_raft_controller`` commands.
+"""
 from __future__ import print_function
 
 import abc
 import os
 import signal
-import six
 import sys
 
 from threading import Lock
+from typing import Any, Optional, Type
+
+from .config import Config
+from .validator import Schema
 
 
-@six.add_metaclass(abc.ABCMeta)
+class AbstractPatroniDaemon(abc.ABC):
-class AbstractPatroniDaemon(object):
+    """A Patroni daemon process.
 
-    def __init__(self, config):
+    .. note::
+
+        When inheriting from :class:`AbstractPatroniDaemon` you are expected to define the methods :func:`_run_cycle`
+        to determine what it should do in each execution cycle, and :func:`_shutdown` to determine what it should do
+        when shutting down.
+
+    :ivar logger: log handler used by this daemon.
+    :ivar config: configuration options for this daemon.
+    """
+
+    def __init__(self, config: Config) -> None:
+        """Set up signal handlers, logging handler and configuration.
+
+        :param config: configuration options for this daemon.
+        """
         from patroni.log import PatroniLogger
 
         self.setup_signal_handlers()
@@ -21,20 +43,44 @@ class AbstractPatroniDaemon(object):
         self.config = config
         AbstractPatroniDaemon.reload_config(self, local=True)
 
-    def sighup_handler(self, *args):
+    def sighup_handler(self, *_: Any) -> None:
+        """Handle SIGHUP signals.
+
+        Flag the daemon as "SIGHUP received".
+        """
         self._received_sighup = True
 
-    def api_sigterm(self):
+    def api_sigterm(self) -> bool:
+        """Guarantee only a single SIGTERM is being processed.
+
+        Flag the daemon as "SIGTERM received" with a lock-based approach.
+
+        :returns: ``True`` if the daemon was flagged as "SIGTERM received".
+        """
+        ret = False
         with self._sigterm_lock:
             if not self._received_sigterm:
                 self._received_sigterm = True
-                return True
+                ret = True
+        return ret
 
-    def sigterm_handler(self, *args):
+    def sigterm_handler(self, *_: Any) -> None:
+        """Handle SIGTERM signals.
+
+        Terminate the daemon process through :func:`api_sigterm`.
+        """
         if self.api_sigterm():
             sys.exit()
 
-    def setup_signal_handlers(self):
+    def setup_signal_handlers(self) -> None:
+        """Set up daemon signal handlers.
+
+        Set up SIGHUP and SIGTERM signal handlers.
+
+        .. note::
+
+            SIGHUP is only handled in non-Windows environments.
+        """
         self._received_sighup = False
         self._sigterm_lock = Lock()
         self._received_sigterm = False
@@ -43,19 +89,34 @@ class AbstractPatroniDaemon(object):
         signal.signal(signal.SIGTERM, self.sigterm_handler)
 
     @property
-    def received_sigterm(self):
+    def received_sigterm(self) -> bool:
+        """If daemon was signaled with SIGTERM."""
         with self._sigterm_lock:
             return self._received_sigterm
 
-    def reload_config(self, sighup=False, local=False):
+    def reload_config(self, sighup: Optional[bool] = False, local: Optional[bool] = False) -> None:
+        """Reload configuration.
+
+        :param sighup: if it is related to a SIGHUP signal.
+                       The sighup parameter could be used in the method overridden in a child class.
+        :param local: will be ``True`` if there are changes in the local configuration file.
+        """
         if local:
             self.logger.reload_config(self.config.get('log', {}))
 
     @abc.abstractmethod
-    def _run_cycle(self):
+    def _run_cycle(self) -> None:
-        """_run_cycle"""
+        """Define what the daemon should do in each execution cycle.
 
-    def run(self):
+        Keep being called in the daemon's main loop until the daemon is eventually terminated.
+        """
+
+    def run(self) -> None:
+        """Run the daemon process.
+
+        Start the logger thread and keep running execution cycles until a SIGTERM is eventually received. Also reload
+        configuration uppon receiving SIGHUP.
+        """
         self.logger.start()
         while not self.received_sigterm:
             if self._received_sighup:
@@ -65,17 +126,29 @@ class AbstractPatroniDaemon(object):
             self._run_cycle()
 
     @abc.abstractmethod
-    def _shutdown(self):
+    def _shutdown(self) -> None:
-        """_shutdown"""
+        """Define what the daemon should do when shutting down."""
 
-    def shutdown(self):
+    def shutdown(self) -> None:
+        """Shut the daemon down when a SIGTERM is received.
+
+        Shut down the daemon process and the logger thread.
+        """
         with self._sigterm_lock:
             self._received_sigterm = True
         self._shutdown()
         self.logger.shutdown()
 
 
-def abstract_main(cls, validator=None):
+def abstract_main(cls: Type[AbstractPatroniDaemon], validator: Optional[Schema] = None) -> None:
+    """Create the main entry point of a given daemon process.
+
+    Expose a basic argument parser, parse the command-line arguments, and run the given daemon process.
+
+    :param cls: a class that should inherit from :class:`AbstractPatroniDaemon`.
+    :param validator: used to validate the daemon configuration schema, if requested by the user through
+        ``--validate-config`` CLI option.
+    """
     import argparse
 
     from .config import Config, ConfigParseError

patroni/dcs/__init__.py

@@ -7,15 +7,15 @@ import logging
 import os
 import pkgutil
 import re
-import six
 import sys
 import time
 
 from collections import defaultdict, namedtuple
 from copy import deepcopy
 from random import randint
-from six.moves.urllib_parse import urlparse, urlunparse, parse_qsl
 from threading import Event, Lock
+from typing import Any, Dict, List, Optional, Union
+from urllib.parse import urlparse, urlunparse, parse_qsl
 
 from ..exceptions import PatroniFatalException
 from ..utils import deep_compare, parse_bool, uri
@@ -375,7 +375,7 @@ class SyncState(namedtuple('SyncState', 'index,leader,sync_standby')):
     """
 
     @staticmethod
-    def from_node(index, value):
+    def from_node(index: Union[str, int], value: Union[str, Dict[str, Any]]) -> 'SyncState':
         """
         >>> SyncState.from_node(1, None).leader is None
         True
@@ -390,27 +390,31 @@ class SyncState(namedtuple('SyncState', 'index,leader,sync_standby')):
         >>> SyncState.from_node(1, {"leader": "leader"}).leader == "leader"
         True
         """
-        if isinstance(value, dict):
+        try:
-            data = value
+            if value and isinstance(value, str):
-        elif value:
+                value = json.loads(value)
-            try:
+            if not isinstance(value, dict):
-                data = json.loads(value)
+                return SyncState.empty(index)
-                if not isinstance(data, dict):
+            return SyncState(index, value.get('leader'), value.get('sync_standby'))
-                    data = {}
+        except (TypeError, ValueError):
-            except (TypeError, ValueError):
+            return SyncState.empty(index)
-                data = {}
+
-        else:
+    @staticmethod
-            data = {}
+    def empty(index: Optional[Union[str, int]] = '') -> 'SyncState':
-        return SyncState(index, data.get('leader'), data.get('sync_standby'))
+        return SyncState(index, None, '')
 
     @property
-    def members(self):
+    def is_empty(self) -> bool:
-        """ Returns sync_standby in list """
+        """:returns: True if /sync key doesn't have a leader"""
-        return self.sync_standby and self.sync_standby.split(',') or []
+        return self.leader is None
 
-    def matches(self, name):
+    @property
-        """
+    def members(self) -> List[str]:
-        Returns if a node name matches one of the nodes in the sync state
+        """:returns: sync_standby as list"""
+        return list(filter(lambda a: a, [s.strip() for s in self.sync_standby.split(',')])) if self.sync_standby else []
+
+    def matches(self, name: str) -> bool:
+        """:returns: True if a node name matches one of the nodes in the sync state (including leader)
 
         >>> s = SyncState(1, 'foo', 'bar,zoo')
         >>> s.matches('foo')
@@ -472,6 +476,10 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_lsn,members,'
             args = args + ({},)
         return super(Cluster, cls).__new__(cls, *args)
 
+    @staticmethod
+    def empty():
+        return Cluster(None, None, None, 0, [], None, SyncState.empty(), None, None, None)
+
     @property
     def leader_name(self):
         return self.leader and self.leader.name
@@ -654,8 +662,7 @@ def catch_return_false_exception(func):
     return wrapper
 
 
-@six.add_metaclass(abc.ABCMeta)
+class AbstractDCS(abc.ABC):
-class AbstractDCS(object):
 
     _INITIALIZE = 'initialize'
     _CONFIG = 'config'
@@ -676,7 +683,7 @@ class AbstractDCS(object):
         """
         self._name = config['name']
         self._base_path = re.sub('/+', '/', '/'.join(['', config.get('namespace', 'service'), config['scope']]))
-        self._citus_group = str(config['group']) if isinstance(config.get('group'), six.integer_types) else None
+        self._citus_group = str(config['group']) if isinstance(config.get('group'), int) else None
         self._set_loop_wait(config.get('loop_wait', 10))
 
         self._ctl = bool(config.get('patronictl', False))
@@ -814,8 +821,8 @@ class AbstractDCS(object):
         if isinstance(groups, Cluster):  # Zookeeper could return a cached version
             cluster = groups
         else:
-            cluster = groups.pop(CITUS_COORDINATOR_GROUP_ID,
+            assert isinstance(groups, dict)
-                                 Cluster(None, None, None, None, [], None, None, None, None, None))
+            cluster = groups.pop(CITUS_COORDINATOR_GROUP_ID, Cluster.empty())
             cluster.workers.update(groups)
         return cluster
 

patroni/dcs/consul.py

@@ -10,9 +10,9 @@ import urllib3
 
 from collections import defaultdict, namedtuple
 from consul import ConsulException, NotFound, base
+from http.client import HTTPException
 from urllib3.exceptions import HTTPError
-from six.moves.urllib.parse import urlencode, urlparse, quote
+from urllib.parse import urlencode, urlparse, quote
-from six.moves.http_client import HTTPException
 
 from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState,\
         TimelineHistory, ReturnFalseException, catch_return_false_exception, citus_group_re
@@ -409,7 +409,7 @@ class Consul(AbstractDCS):
         try:
             return loader(path)
         except NotFound:
-            return Cluster(None, None, None, None, [], None, None, None, None, None)
+            return Cluster.empty()
         except Exception:
             logger.exception('get_cluster')
             raise ConsulError('Consul is not responding properly')

patroni/dcs/etcd.py

@@ -6,7 +6,6 @@ import logging
 import os
 import urllib3.util.connection
 import random
-import six
 import socket
 import time
 
@@ -14,12 +13,13 @@ from collections import defaultdict
 from copy import deepcopy
 from dns.exception import DNSException
 from dns import resolver
+from http.client import HTTPException
+from queue import Queue
+from threading import Thread
+from typing import List, Optional
+from urllib.parse import urlparse
 from urllib3 import Timeout
 from urllib3.exceptions import HTTPError, ReadTimeoutError, ProtocolError
-from six.moves.queue import Queue
-from six.moves.http_client import HTTPException
-from six.moves.urllib_parse import urlparse
-from threading import Thread
 
 from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState,\
         TimelineHistory, ReturnFalseException, catch_return_false_exception, citus_group_re
@@ -86,8 +86,7 @@ class DnsCachingResolver(Thread):
             return []
 
 
-@six.add_metaclass(abc.ABCMeta)
+class AbstractEtcdClientWithFailover(abc.ABC, etcd.Client):
-class AbstractEtcdClientWithFailover(etcd.Client):
 
     def __init__(self, config, dns_resolver, cache_ttl=300):
         self._dns_resolver = dns_resolver
@@ -100,7 +99,6 @@ class AbstractEtcdClientWithFailover(etcd.Client):
         # Workaround for the case when https://github.com/jplana/python-etcd/pull/196 is not applied
         self.http.connection_pool_kw.pop('ssl_version', None)
         self._config = config
-        self._initial_machines_cache = []
         self._load_machines_cache()
         self._allow_reconnect = True
         # allow passing retry argument to api_execute in params
@@ -170,19 +168,12 @@ class AbstractEtcdClientWithFailover(etcd.Client):
         base_uri, cache = self._base_uri, self._machines_cache
         return ([base_uri] if base_uri in cache else []) + [machine for machine in cache if machine != base_uri]
 
-    @property
+    def _get_machines_list(self, machines_cache: List[str]) -> List[str]:
-    def machines(self):
+        """Gets list of members from Etcd cluster using API
-        """Original `machines` method(property) of `etcd.Client` class raise exception
-        when it failed to get list of etcd cluster members. This method is being called
-        only when request failed on one of the etcd members during `api_execute` call.
-        For us it's more important to execute original request rather then get new topology
-        of etcd cluster. So we will catch this exception and return empty list of machines.
-        Later, during next `api_execute` call we will forcefully update machines_cache.
 
-        Also this method implements the same timeout-retry logic as `api_execute`, because
+        :param machines_cache: initial list of Etcd members
-        the original method was retrying 2 times with the `read_timeout` on each node."""
+        :returns: list of clientURLs retrieved from Etcd cluster
-
+        :raises EtcdConnectionFailed: if failed"""
-        machines_cache = self.machines_cache
         kwargs = self._prepare_get_members(len(machines_cache))
 
         for base_uri in machines_cache:
@@ -200,6 +191,22 @@ class AbstractEtcdClientWithFailover(etcd.Client):
 
         raise etcd.EtcdConnectionFailed('No more machines in the cluster')
 
+    @property
+    def machines(self) -> List[str]:
+        """Original `machines` method(property) of `etcd.Client` class raise exception
+        when it failed to get list of etcd cluster members. This method is being called
+        only when request failed on one of the etcd members during `api_execute` call.
+        For us it's more important to execute original request rather then get new topology
+        of etcd cluster. So we will catch this exception and return empty list of machines.
+        Later, during next `api_execute` call we will forcefully update machines_cache.
+
+        Also this method implements the same timeout-retry logic as `api_execute`, because
+        the original method was retrying 2 times with the `read_timeout` on each node.
+
+        After the next refactoring the whole logic was moved to the _get_machines_list() method."""
+
+        return self._get_machines_list(self.machines_cache)
+
     def set_read_timeout(self, timeout):
         self._read_timeout = timeout
 
@@ -369,36 +376,46 @@ class AbstractEtcdClientWithFailover(etcd.Client):
         # enforce resolving dns name,they might get new ips
         self._update_dns_cache(self._dns_resolver.remove, machines_cache)
 
-        # The etcd cluster could change its topology over time and depending on how we resolve the initial
+        # after filling up the initial list of machines_cache we should ask etcd-cluster about actual list
-        # topology (list of hosts in the Patroni config or DNS records, A or SRV) we might get into the situation
+        ret = self._refresh_machines_cache(machines_cache)
-        # the the real topology doesn't match anymore with the topology resolved from the configuration file.
-        # In case if the "initial" topology is the same as before we will not override the `_machines_cache`.
-        ret = set(machines_cache) != set(self._initial_machines_cache)
-        if ret:
-            self._initial_machines_cache = self._machines_cache = machines_cache
-
-            # After filling up the initial list of machines_cache we should ask etcd-cluster about actual list
-            self._refresh_machines_cache(True)
 
         self._update_machines_cache = False
         return ret
 
-    def _refresh_machines_cache(self, updating_cache=False):
+    def _refresh_machines_cache(self, machines_cache: Optional[List[str]] = None) -> bool:
+        """Get etcd cluster topology using Etcd API and put it to self._machines_cache
+
+        :param machines_cache: the list of nodes we want to run through executing API request
+                               in addition to values stored in the self._machines_cache
+        :returns: `True` if self._machines_cache was updated with new values
+        :raises EtcdException: if failed to get topology and `machines_cache` was specified.
+
+        The self._machines_cache will not be updated if nodes from the list are
+        not accessible or if they are not returning correct results."""
+
         if self._use_proxies:
-            self._machines_cache = self._get_machines_cache_from_config()
+            value = self._get_machines_cache_from_config()
         else:
             try:
-                self._machines_cache = self.machines
+                # we want to go through the list obtained from the config file + last known health topology
+                value = self._get_machines_list(list(set((machines_cache or []) + self.machines_cache)))
             except etcd.EtcdConnectionFailed:
-                if updating_cache:
+                value = []
-                    raise etcd.EtcdException("Could not get the list of servers, "
+
-                                             "maybe you provided the wrong "
+        if value:
-                                             "host(s) to connect to?")
+            ret = set(self._machines_cache) != set(value)
-                return
+            self._machines_cache = value
+        elif machines_cache:  # we are just starting or all nodes were not available at some point
+            raise etcd.EtcdException("Could not get the list of servers, "
+                                     "maybe you provided the wrong "
+                                     "host(s) to connect to?")
+        else:
+            return False
 
         if self._base_uri not in self._machines_cache:
             self.set_base_uri(self._machines_cache[0])
         self._machines_cache_updated = time.time()
+        return ret
 
     def set_base_uri(self, value):
         if self._base_uri != value:
@@ -496,12 +513,12 @@ class AbstractEtcd(AbstractDCS):
             default_port = config.pop('port', 2379)
             protocol = config.get('protocol', 'http')
 
-            if isinstance(hosts, six.string_types):
+            if isinstance(hosts, str):
                 hosts = hosts.split(',')
 
             config['hosts'] = []
             for value in hosts:
-                if isinstance(value, six.string_types):
+                if isinstance(value, str):
                     config['hosts'].append(uri(protocol, split_host_port(value.strip(), default_port)))
         elif 'host' in config:
             host, port = split_host_port(config['host'], 2379)
@@ -687,7 +704,7 @@ class Etcd(AbstractEtcd):
         try:
             cluster = loader(path)
         except etcd.EtcdKeyNotFound:
-            cluster = Cluster(None, None, None, None, [], None, None, None, None, None)
+            cluster = Cluster.empty()
         except Exception as e:
             self._handle_exception(e, 'get_cluster', raise_ex=EtcdError('Etcd is not responding properly'))
         self._has_failed = False

patroni/dcs/etcd3.py

@@ -4,7 +4,6 @@ import etcd
 import json
 import logging
 import os
-import six
 import socket
 import sys
 import time
@@ -177,20 +176,6 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
         self.version_prefix = '/v3beta'
         super(Etcd3Client, self).__init__(config, dns_resolver, cache_ttl)
 
-        if six.PY2:  # pragma: no cover
-            # Old grpc-gateway sometimes sends double 'transfer-encoding: chunked' headers,
-            # what breaks the old (python2.7) httplib.HTTPConnection (it closes the socket).
-            def dedup_addheader(httpm, key, value):
-                prev = httpm.dict.get(key)
-                if prev is None:
-                    httpm.dict[key] = value
-                elif key != 'transfer-encoding' or prev != value:
-                    combined = ", ".join((prev, value))
-                    httpm.dict[key] = combined
-
-            import httplib
-            httplib.HTTPMessage.addheader = dedup_addheader
-
         try:
             self.authenticate()
         except AuthFailed as e:

patroni/dcs/exhibitor.py

@@ -64,9 +64,7 @@ class Exhibitor(ZooKeeper):
     def __init__(self, config):
         interval = config.get('poll_interval', 300)
         self._ensemble_provider = ExhibitorEnsembleProvider(config['hosts'], config['port'], poll_interval=interval)
-        config = config.copy()
+        super(Exhibitor, self).__init__({**config, 'hosts': self._ensemble_provider.zookeeper_hosts})
-        config['hosts'] = self._ensemble_provider.zookeeper_hosts
-        super(Exhibitor, self).__init__(config)
 
     def _load_cluster(self, path, loader):
         if self._ensemble_provider.poll():

patroni/dcs/kubernetes.py

@@ -7,7 +7,6 @@ import logging
 import os
 import random
 import socket
-import six
 import tempfile
 import time
 import urllib3
@@ -15,10 +14,10 @@ import yaml
 
 from collections import defaultdict
 from copy import deepcopy
-from urllib3 import Timeout
+from http.client import HTTPException
-from urllib3.exceptions import HTTPError
-from six.moves.http_client import HTTPException
 from threading import Condition, Lock, Thread
+from typing import Any, Dict, List, Optional
+from urllib3.exceptions import HTTPError
 
 from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState,\
         TimelineHistory, CITUS_COORDINATOR_GROUP_ID, citus_group_re
@@ -176,7 +175,7 @@ class K8sObject(object):
         if isinstance(value, dict):
             # we know that `annotations` and `labels` are dicts and therefore don't want to convert them into K8sObject
             return value if parent in {'annotations', 'labels'} and \
-                    all(isinstance(v, six.string_types) for v in value.values()) else cls(value)
+                    all(isinstance(v, str) for v in value.values()) else cls(value)
         elif isinstance(value, list):
             return [cls._wrap(None, v) for v in value]
         else:
@@ -220,7 +219,7 @@ class K8sClient(object):
 
         _API_URL_PREFIX = '/api/v1/namespaces/'
 
-        def __init__(self, bypass_api_service=False):
+        def __init__(self, bypass_api_service: Optional[bool] = False) -> None:
             self._bypass_api_service = bypass_api_service
             self.pool_manager = urllib3.PoolManager(**k8s_config.pool_config)
             self._base_uri = k8s_config.server
@@ -376,7 +375,7 @@ class K8sClient(object):
             api_servers = len(api_servers_cache)
 
             if timeout:
-                if isinstance(timeout, six.integer_types + (float,)):
+                if isinstance(timeout, (int, float)):
                     timeout = urllib3.Timeout(total=timeout)
                 elif isinstance(timeout, tuple) and len(timeout) == 2:
                     timeout = urllib3.Timeout(connect=timeout[0], read=timeout[1])
@@ -488,11 +487,15 @@ class KubernetesRetriableException(k8s_client.rest.ApiException):
 
 
 class CoreV1ApiProxy(object):
+    """Proxy class to work with k8s_client.CoreV1Api() object"""
 
-    def __init__(self, use_endpoints=False, bypass_api_service=False):
+    _DEFAULT_RETRIABLE_HTTP_CODES = frozenset([500, 503, 504])
+
+    def __init__(self, use_endpoints: Optional[bool] = False, bypass_api_service: Optional[bool] = False) -> None:
         self._api_client = k8s_client.ApiClient(bypass_api_service)
         self._core_v1_api = k8s_client.CoreV1Api(self._api_client)
         self._use_endpoints = bool(use_endpoints)
+        self._retriable_http_codes = set(self._DEFAULT_RETRIABLE_HTTP_CODES)
 
     def configure_timeouts(self, loop_wait, retry_timeout, ttl):
         # Normally every loop_wait seconds we should have receive something from the socket.
@@ -504,10 +507,21 @@ class CoreV1ApiProxy(object):
         self._api_client.set_read_timeout(retry_timeout)
         self._api_client.set_api_servers_cache_ttl(loop_wait)
 
+    def configure_retriable_http_codes(self, retriable_http_codes: List[int]) -> None:
+        self._retriable_http_codes = self._DEFAULT_RETRIABLE_HTTP_CODES | set(retriable_http_codes)
+
     def refresh_api_servers_cache(self):
         self._api_client.refresh_api_servers_cache()
 
-    def __getattr__(self, func):
+    def __getattr__(self, func: str):
+        """Intercepts calls to `CoreV1Api` methods.
+
+        Handles two important cases:
+        1. Depending on whether Patroni is configured to work with `ConfigMaps` or `Endpoints`
+           it remaps "virtual" method names from `*_kind` to `*_endpoints` or `*_config_map`.
+        2. It handles HTTP error codes and raises `KubernetesRetriableException`
+           if the given error is supposed to be handled with retry."""
+
         if func.endswith('_kind'):
             func = func[:-4] + ('endpoints' if self._use_endpoints else 'config_map')
 
@@ -515,7 +529,7 @@ class CoreV1ApiProxy(object):
             try:
                 return getattr(self._core_v1_api, func)(*args, **kwargs)
             except k8s_client.rest.ApiException as e:
-                if e.status in (500, 503, 504) or e.headers and 'retry-after' in e.headers:  # XXX
+                if e.status in self._retriable_http_codes or e.headers and 'retry-after' in e.headers:
                     raise KubernetesRetriableException(e)
                 raise
         return wrapper
@@ -560,7 +574,7 @@ class ObjectCache(Thread):
             raise
 
     def _watch(self, resource_version):
-        return self._func(_request_timeout=(self._retry.deadline, Timeout.DEFAULT_TIMEOUT),
+        return self._func(_request_timeout=(self._retry.deadline, urllib3.Timeout.DEFAULT_TIMEOUT),
                           _preload_content=False, watch=True, resource_version=resource_version)
 
     def set(self, name, value):
@@ -698,8 +712,7 @@ class Kubernetes(AbstractDCS):
         self._namespace = config.get('namespace') or 'default'
         self._role_label = config.get('role_label', 'role')
         self._ca_certs = os.environ.get('PATRONI_KUBERNETES_CACERT', config.get('cacert')) or SERVICE_CERT_FILENAME
-        config['namespace'] = ''
+        super(Kubernetes, self).__init__({**config, 'namespace': ''})
-        super(Kubernetes, self).__init__(config)
         if self._citus_group:
             self._labels[self._CITUS_LABEL] = self._citus_group
 
@@ -776,10 +789,24 @@ class Kubernetes(AbstractDCS):
     def set_retry_timeout(self, retry_timeout):
         self._retry.deadline = retry_timeout
 
-    def reload_config(self, config):
+    def reload_config(self, config: Dict[str, Any]) -> None:
+        """Handles dynamic config changes.
+
+        Either cause by changes in the local configuration file + SIGHUP or by changes of dynamic configuration"""
+
         super(Kubernetes, self).reload_config(config)
         self._api.configure_timeouts(self.loop_wait, self._retry.deadline, self.ttl)
 
+        # retriable_http_codes supposed to be either int, list of integers or comma-separated string with integers.
+        retriable_http_codes = config.get('retriable_http_codes', [])
+        if not isinstance(retriable_http_codes, list):
+            retriable_http_codes = [c.strip() for c in str(retriable_http_codes).split(',')]
+
+        try:
+            self._api.configure_retriable_http_codes([int(c) for c in retriable_http_codes])
+        except Exception as e:
+            logger.warning('Invalid value of retriable_http_codes = %s: %r', config['retriable_http_codes'], e)
+
     @staticmethod
     def member(pod):
         annotations = pod.metadata.annotations or {}

patroni/dcs/raft.py

@@ -383,7 +383,7 @@ class Raft(AbstractDCS):
     def _cluster_loader(self, path):
         response = self._sync_obj.get(path, recursive=True)
         if not response:
-            return Cluster(None, None, None, None, [], None, None, None, None, None)
+            return Cluster.empty()
         nodes = {key[len(path):]: value for key, value in response.items()}
         return self._cluster_from_nodes(nodes)
 

patroni/dcs/zookeeper.py

@@ -1,7 +1,6 @@
 import json
 import logging
 import select
-import six
 import time
 
 from kazoo.client import KazooClient, KazooState, KazooRetry
@@ -63,10 +62,8 @@ class PatroniSequentialThreadingHandler(SequentialThreadingHandler):
 
         try:
             return super(PatroniSequentialThreadingHandler, self).select(*args, **kwargs)
-        except IOError as e:
-            raise (select.error(e.errno, e.strerror) if six.PY2 else e)
         except (TypeError, ValueError) as e:
-            raise (e if six.PY2 and isinstance(e, TypeError) else select.error(9, str(e)))
+            raise select.error(9, str(e))
 
 
 class PatroniKazooClient(KazooClient):

patroni/ha.py

@@ -2,7 +2,6 @@ import datetime
 import functools
 import json
 import logging
-import six
 import sys
 import time
 import uuid
@@ -14,7 +13,7 @@ from threading import RLock
 from . import psycopg
 from .async_executor import AsyncExecutor, CriticalTask
 from .exceptions import DCSError, PostgresConnectionException, PatroniFatalException
-from .postgresql import ACTION_ON_START, ACTION_ON_ROLE_CHANGE
+from .postgresql.callback_executor import CallbackAction
 from .postgresql.misc import postgres_version_to_int
 from .postgresql.rewind import Rewind
 from .utils import polling_loop, tzutc, is_standby_cluster as _is_standby_cluster, parse_int
@@ -567,7 +566,7 @@ class Ha(object):
                 self._rewind.trigger_check_diverged_lsn()
             elif role == 'standby_leader' and self.state_handler.role != role:
                 self.state_handler.set_role(role)
-                self.state_handler.call_nowait(ACTION_ON_ROLE_CHANGE)
+                self.state_handler.call_nowait(CallbackAction.ON_ROLE_CHANGE)
 
         return follow_reason
 
@@ -591,7 +590,7 @@ class Ha(object):
         """
         if self.is_synchronous_mode():
             sync_node_count = self.patroni.config['synchronous_node_count']
-            current = self.cluster.sync.leader and self.cluster.sync.members or []
+            current = [] if self.cluster.sync.is_empty else self.cluster.sync.members
             picked, allow_promote = self.state_handler.sync_handler.current_state(self.cluster, sync_node_count,
                                                                                   self.patroni.config[
                                                                                       'maximum_lag_on_syncnode'])
@@ -626,7 +625,7 @@ class Ha(object):
                         cluster = self.dcs.get_cluster()
                     except DCSError:
                         return logger.warning("Could not get cluster state from DCS during process_sync_replication()")
-                    if cluster.sync.leader and cluster.sync.leader != self.state_handler.name:
+                    if not cluster.sync.is_empty and cluster.sync.leader != self.state_handler.name:
                         logger.info("Synchronous replication key updated by someone else")
                         return
                     if not self.dcs.write_sync_state(self.state_handler.name, allow_promote, index=cluster.sync.index):
@@ -634,7 +633,7 @@ class Ha(object):
                         return
                     logger.info("Synchronous standby status assigned to %s", allow_promote)
         else:
-            if self.cluster.sync.leader and self.dcs.delete_sync_state(index=self.cluster.sync.index):
+            if not self.cluster.sync.is_empty and self.dcs.delete_sync_state(index=self.cluster.sync.index):
                 logger.info("Disabled synchronous replication")
             self.state_handler.sync_handler.set_synchronous_standby_names([])
 
@@ -880,7 +879,7 @@ class Ha(object):
                 not_allowed_reason = st.failover_limitation()
                 if not_allowed_reason:
                     logger.info('Member %s is %s', st.member.name, not_allowed_reason)
-                elif not isinstance(st.wal_position, six.integer_types):
+                elif not isinstance(st.wal_position, int):
                     logger.info('Member %s does not report wal_position', st.member.name)
                 elif cluster_lsn and st.wal_position < cluster_lsn or\
                         not cluster_lsn and self.is_lagging(st.wal_position):
@@ -995,7 +994,7 @@ class Ha(object):
         all_known_members += self.cluster.members
 
         # When in sync mode, only last known primary and sync standby are allowed to promote automatically.
-        if self.is_synchronous_mode() and self.cluster.sync and self.cluster.sync.leader:
+        if self.is_synchronous_mode() and not self.cluster.sync.is_empty:
             if not self.cluster.sync.matches(self.state_handler.name):
                 return False
             # pick between synchronous candidates so we minimize unnecessary failovers/demotions
@@ -1502,7 +1501,7 @@ class Ha(object):
         self.dcs.set_config_value(json.dumps(self.patroni.config.dynamic_configuration, separators=(',', ':')))
         self.dcs.take_leader()
         self.set_is_leader(True)
-        self.state_handler.call_nowait(ACTION_ON_START)
+        self.state_handler.call_nowait(CallbackAction.ON_START)
         self.load_cluster_from_dcs()
 
         return 'initialized a new cluster'
@@ -1700,7 +1699,7 @@ class Ha(object):
                 if not self.state_handler.cb_called:
                     if not self.state_handler.is_leader():
                         self._rewind.trigger_check_diverged_lsn()
-                    self.state_handler.call_nowait(ACTION_ON_START)
+                    self.state_handler.call_nowait(CallbackAction.ON_START)
                 if create_slots and self.cluster.leader:
                     err = self._async_executor.try_run_async('copy_logical_slots',
                                                              self.state_handler.slots_handler.copy_logical_slots,

patroni/log.py

@@ -5,7 +5,7 @@ import sys
 from copy import deepcopy
 from logging.handlers import RotatingFileHandler
 from patroni.utils import deep_compare
-from six.moves.queue import Queue, Full
+from queue import Queue, Full
 from threading import Lock, Thread
 
 _LOGGER = logging.getLogger(__name__)

patroni/postgresql/__init__.py

@@ -3,7 +3,6 @@ import os
 import re
 import shlex
 import shutil
-import six
 import subprocess
 import time
 
@@ -13,9 +12,10 @@ from datetime import datetime
 from dateutil import tz
 from psutil import TimeoutExpired
 from threading import current_thread, Lock
+from typing import Optional
 
 from .bootstrap import Bootstrap
-from .callback_executor import CallbackExecutor
+from .callback_executor import CallbackAction, CallbackExecutor
 from .cancellable import CancellableSubprocess
 from .config import ConfigHandler, mtime
 from .connection import Connection, get_connection_cursor
@@ -25,19 +25,13 @@ from .postmaster import PostmasterProcess
 from .slots import SlotsHandler
 from .sync import SyncHandler
 from .. import psycopg
+from ..dcs import Member
 from ..exceptions import PostgresConnectionException
 from ..utils import Retry, RetryFailedError, polling_loop, data_directory_is_empty, parse_int
 
 
 logger = logging.getLogger(__name__)
 
-ACTION_ON_START = "on_start"
-ACTION_ON_STOP = "on_stop"
-ACTION_ON_RESTART = "on_restart"
-ACTION_ON_RELOAD = "on_reload"
-ACTION_ON_ROLE_CHANGE = "on_role_change"
-ACTION_NOOP = "noop"
-
 STATE_RUNNING = 'running'
 STATE_REJECT = 'rejecting connections'
 STATE_NO_RESPONSE = 'not responding'
@@ -207,7 +201,10 @@ class Postgresql(object):
     def _version_file_exists(self):
         return not self.data_directory_empty() and os.path.isfile(self._version_file)
 
-    def get_major_version(self):
+    def get_major_version(self) -> int:
+        """Reads major version from PG_VERSION file
+
+        :returns: major PostgreSQL version in integer format or 0 in case of missing file or errors"""
         if self._version_file_exists():
             try:
                 with open(self._version_file) as f:
@@ -475,7 +472,7 @@ class Postgresql(object):
                         return prev
             except Exception as e:
                 logger.error('Exception when parsing WAL pg_%sdump output: %r', self.wal_name, e)
-            if isinstance(checkpoint_lsn, six.integer_types):
+            if isinstance(checkpoint_lsn, int):
                 return checkpoint_lsn
 
     def is_running(self):
@@ -496,20 +493,22 @@ class Postgresql(object):
     def cb_called(self):
         return self.__cb_called
 
-    def call_nowait(self, cb_name):
+    def call_nowait(self, cb_type: CallbackAction) -> None:
-        """ pick a callback command and call it without waiting for it to finish """
+        """pick a callback command and call it without waiting for it to finish """
         if self.bootstrapping:
             return
-        if cb_name in (ACTION_ON_START, ACTION_ON_STOP, ACTION_ON_RESTART, ACTION_ON_ROLE_CHANGE):
+        if cb_type in (CallbackAction.ON_START, CallbackAction.ON_STOP,
+                       CallbackAction.ON_RESTART, CallbackAction.ON_ROLE_CHANGE):
             self.__cb_called = True
 
-        if self.callback and cb_name in self.callback:
+        if self.callback and cb_type in self.callback:
-            cmd = self.callback[cb_name]
+            cmd = self.callback[cb_type]
+            role = 'master' if self.role == 'promoted' else self.role
             try:
-                cmd = shlex.split(self.callback[cb_name]) + [cb_name, self.role, self.scope]
+                cmd = shlex.split(self.callback[cb_type]) + [cb_type, role, self.scope]
                 self._callback_executor.call(cmd)
             except Exception:
-                logger.exception('callback %s %s %s %s failed', cmd, cb_name, self.role, self.scope)
+                logger.exception('callback %s %r %s %s failed', cmd, cb_type, role, self.scope)
 
     @property
     def role(self):
@@ -562,7 +561,8 @@ class Postgresql(object):
         Waits for postmaster to open ports or terminate so pg_isready can be used to check startup completion
         or failure.
 
-        :returns: True if start was initiated and postmaster ports are open, False if start failed"""
+        :returns: True if start was initiated and postmaster ports are open,
+                  False if start failed, and None if postgres is still starting up"""
         # make sure we close all connections established against
         # the former node, otherwise, we might get a stalled one
         # after kill -9, which would report incorrect data to
@@ -575,7 +575,7 @@ class Postgresql(object):
             return True
 
         if not block_callbacks:
-            self.__cb_pending = ACTION_ON_START
+            self.__cb_pending = CallbackAction.ON_START
 
         self.set_role(role or self.get_postgres_role_from_data_directory())
 
@@ -583,8 +583,8 @@ class Postgresql(object):
         self._pending_restart = False
 
         try:
-            if not self._major_version:
+            if not self.ensure_major_version_is_known():
-                self.configure_server_parameters()
+                return None
             configuration = self.config.effective_configuration
         except Exception:
             return None
@@ -677,7 +677,7 @@ class Postgresql(object):
             if not block_callbacks:
                 self.set_state('stopped')
                 if pg_signaled:
-                    self.call_nowait(ACTION_ON_STOP)
+                    self.call_nowait(CallbackAction.ON_STOP)
         else:
             logger.warning('pg_ctl stop failed')
             self.set_state('stop failed')
@@ -769,7 +769,7 @@ class Postgresql(object):
     def reload(self, block_callbacks=False):
         ret = self.pg_ctl('reload')
         if ret and not block_callbacks:
-            self.call_nowait(ACTION_ON_RELOAD)
+            self.call_nowait(CallbackAction.ON_RELOAD)
         return ret
 
     def check_for_startup(self):
@@ -805,7 +805,7 @@ class Postgresql(object):
             self.config.save_configuration_files(True)
             # TODO: __cb_pending can be None here after PostgreSQL restarts on its own. Do we want to call the callback?
             # Previously we didn't even notice.
-            action = self.__cb_pending or ACTION_ON_START
+            action = self.__cb_pending or CallbackAction.ON_START
             self.call_nowait(action)
             self.__cb_pending = None
 
@@ -837,7 +837,7 @@ class Postgresql(object):
         """
         self.set_state('restarting')
         if not block_callbacks:
-            self.__cb_pending = ACTION_ON_RESTART
+            self.__cb_pending = CallbackAction.ON_RESTART
         ret = self.stop(block_callbacks=True, before_shutdown=before_shutdown)\
             and self.start(timeout, task, True, role, after_start)
         if not ret and not self.is_starting():
@@ -865,8 +865,7 @@ class Postgresql(object):
         # Don't try to call pg_controldata during backup restore
         if self._version_file_exists() and self.state != 'creating replica':
             try:
-                env = os.environ.copy()
+                env = {**os.environ, 'LANG': 'C', 'LC_ALL': 'C'}
-                env.update(LANG='C', LC_ALL='C')
                 data = subprocess.check_output([self.pgcommand('pg_controldata'), self._data_dir], env=env)
                 if data:
                     data = filter(lambda e: ':' in e, data.decode('utf-8').splitlines())
@@ -878,8 +877,7 @@ class Postgresql(object):
 
     def waldump(self, timeline, lsn, limit):
         cmd = self.pgcommand('pg_{0}dump'.format(self.wal_name))
-        env = os.environ.copy()
+        env = {**os.environ, 'LANG': 'C', 'LC_ALL': 'C', 'PGDATA': self._data_dir}
-        env.update(LANG='C', LC_ALL='C', PGDATA=self._data_dir)
         try:
             waldump = subprocess.Popen([cmd, '-t', str(timeline), '-s', str(lsn), '-n', str(limit)],
                                        stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
@@ -931,7 +929,27 @@ class Postgresql(object):
             except Exception:
                 logger.exception('Failed to read and parse %s', (history_path,))
 
-    def follow(self, member, role='replica', timeout=None, do_reload=False):
+    def follow(self,
+               member: Member,
+               role: Optional[str] = 'replica',
+               timeout: Optional[float] = None,
+               do_reload: Optional[bool] = False) -> Optional[bool]:
+        """Reconfigure postgres to follow a new member or use different recovery parameters.
+
+        Method may call `on_role_change` callback if role is changing.
+
+        :param member: The member to follow
+        :param role: The desired role, normally 'replica', but could also be a 'standby_leader'
+        :param timeout: start timeout, how long should the `start()` method wait for postgres accepting connections
+        :param do_reload: indicates that after updating postgresql.conf we just need to do a reload instead of restart
+
+        :returns: True - if restart/reload were successfully performed,
+                  False - if restart/reload failed
+                  None - if nothing was done or if Postgres is still in starting state after `timeout` seconds."""
+
+        if not self.ensure_major_version_is_known():
+            return None
+
         recovery_params = self.config.build_recovery_params(member)
         self.config.write_recovery_conf(recovery_params)
 
@@ -942,7 +960,7 @@ class Postgresql(object):
         change_role = self.cb_called and (self.role in ('master', 'primary', 'demoted') or
                                           not {'standby_leader', 'replica'} - {self.role, role})
         if change_role:
-            self.__cb_pending = ACTION_NOOP
+            self.__cb_pending = CallbackAction.NOOP
 
         ret = True
         if self.is_running():
@@ -958,7 +976,7 @@ class Postgresql(object):
 
         if change_role:
             # TODO: postpone this until start completes, or maybe do even earlier
-            self.call_nowait(ACTION_ON_ROLE_CHANGE)
+            self.call_nowait(CallbackAction.ON_ROLE_CHANGE)
         return ret
 
     def _wait_promote(self, wait_seconds):
@@ -1011,7 +1029,7 @@ class Postgresql(object):
             self.set_role('promoted')
             if on_success is not None:
                 on_success()
-            self.call_nowait(ACTION_ON_ROLE_CHANGE)
+            self.call_nowait(CallbackAction.ON_ROLE_CHANGE)
             ret = self._wait_promote(wait_seconds)
         return ret
 
@@ -1055,7 +1073,15 @@ class Postgresql(object):
     def configure_server_parameters(self):
         self._major_version = self.get_major_version()
         self.config.setup_server_parameters()
-        return True
+
+    def ensure_major_version_is_known(self) -> bool:
+        """Calls configure_server_parameters() if `_major_version` is not known
+
+        :returns: `True` if `_major_version` is set, otherwise `False`"""
+
+        if not self._major_version:
+            self.configure_server_parameters()
+        return self._major_version > 0
 
     def pg_wal_realpath(self):
         """Returns a dict containing the symlink (key) and target (value) for the wal directory"""
@@ -1148,8 +1174,7 @@ class Postgresql(object):
             2. sync replication slots, because it might happen that slots were removed
             3. get new 'Database system identifier' to make sure that it wasn't changed
         """
-        if not self._major_version:
+        self.ensure_major_version_is_known()
-            self.configure_server_parameters()
         self.slots_handler.schedule()
         self.citus_handler.schedule_cache_rebuild()
         self._sysid = None

patroni/postgresql/bootstrap.py

@@ -4,8 +4,6 @@ import shlex
 import tempfile
 import time
 
-from six import string_types
-
 from ..dcs import RemoteMember
 from ..psycopg import quote_ident, quote_literal
 from ..utils import deep_compare
@@ -43,11 +41,11 @@ class Bootstrap(object):
                     user_options.append('--{0}={1}'.format(k, v))
         elif isinstance(options, list):
             for opt in options:
-                if isinstance(opt, string_types) and option_is_allowed(opt):
+                if isinstance(opt, str) and option_is_allowed(opt):
                     user_options.append('--{0}'.format(opt))
                 elif isinstance(opt, dict):
                     keys = list(opt.keys())
-                    if len(keys) != 1 or not isinstance(opt[keys[0]], string_types) or not option_is_allowed(keys[0]):
+                    if len(keys) != 1 or not isinstance(opt[keys[0]], str) or not option_is_allowed(keys[0]):
                         error_handler('Error when parsing {0} key-value option {1}: only one key-value is allowed'
                                       ' and value should be a string'.format(tool, opt[keys[0]]))
                     user_options.append('--{0}={1}'.format(keys[0], opt[keys[0]]))
@@ -59,7 +57,7 @@ class Bootstrap(object):
         return user_options
 
     def _initdb(self, config):
-        self._postgresql.set_state('initalizing new cluster')
+        self._postgresql.set_state('initializing new cluster')
         not_allowed_options = ('pgdata', 'nosync', 'pwfile', 'sync-only', 'version')
 
         def error_handler(e):

patroni/postgresql/callback_executor.py

@@ -1,22 +1,60 @@
 import logging
 
-from patroni.postgresql.cancellable import CancellableExecutor
+from enum import Enum
 from threading import Condition, Thread
+from typing import List
+
+from .cancellable import CancellableExecutor, CancellableSubprocess
 
 logger = logging.getLogger(__name__)
 
 
+class CallbackAction(str, Enum):
+    NOOP = "noop"
+    ON_START = "on_start"
+    ON_STOP = "on_stop"
+    ON_RESTART = "on_restart"
+    ON_RELOAD = "on_reload"
+    ON_ROLE_CHANGE = "on_role_change"
+
+    def __repr__(self):
+        return self.value
+
+
+class OnReloadExecutor(CancellableSubprocess):
+
+    def call_nowait(self, cmd: List[str]) -> None:
+        """Run one `on_reload` callback at most.
+
+        To achieve it we always kill already running command including child processes."""
+        self.cancel(kill=True)
+        self._kill_children()
+        with self._lock:
+            self._start_process(cmd, close_fds=True)
+
+
 class CallbackExecutor(CancellableExecutor, Thread):
 
     def __init__(self):
         CancellableExecutor.__init__(self)
         Thread.__init__(self)
         self.daemon = True
+        self._on_reload_executor = OnReloadExecutor()
         self._cmd = None
         self._condition = Condition()
         self.start()
 
-    def call(self, cmd):
+    def call(self, cmd: List[str]) -> None:
+        """Executes one callback at a time.
+
+        Already running command is killed (including child processes).
+        If it couldn't be killed we wait until it finishes.
+
+        :param cmd: command to be executed"""
+
+        if cmd[-3] == CallbackAction.ON_RELOAD:
+            return self._on_reload_executor.call_nowait(cmd)
+
         self._kill_process()
         with self._condition:
             self._cmd = cmd

patroni/postgresql/citus.py

@@ -2,8 +2,8 @@ import logging
 import re
 import time
 
-from six.moves.urllib_parse import urlparse
 from threading import Condition, Event, Thread
+from urllib.parse import urlparse
 
 from .connection import Connection
 from ..dcs import CITUS_COORDINATOR_GROUP_ID
@@ -196,17 +196,12 @@ class CitusHandler(Thread):
             return i, task
 
     def update_node(self, task):
-        if task.group == CITUS_COORDINATOR_GROUP_ID:
+        if task.nodeid is not None:
-            return self.query("SELECT pg_catalog.citus_set_coordinator_host(%s, %s, 'primary', 'default')",
-                              task.host, task.port)
-
-        if task.nodeid is None and task.event != 'before_demote':
-            task.nodeid = self.query("SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default')",
-                                     task.host, task.port, task.group).fetchone()[0]
-        elif task.nodeid is not None:
-            # XXX: statement_timeout?
             self.query('SELECT pg_catalog.citus_update_node(%s, %s, %s, true, %s)',
                        task.nodeid, task.host, task.port, task.cooldown)
+        elif task.event != 'before_demote':
+            task.nodeid = self.query("SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default')",
+                                     task.host, task.port, task.group).fetchone()[0]
 
     def process_task(self, task):
         """Updates a single row in `pg_dist_node` table, optionally in a transaction.
@@ -361,6 +356,11 @@ class CitusHandler(Thread):
                     cur.execute("INSERT INTO pg_catalog.pg_dist_authinfo VALUES"
                                 "(0, pg_catalog.current_user(), %s)",
                                 (self._postgresql.config.format_dsn(params),))
+
+                if self.is_coordinator():
+                    r = urlparse(self._postgresql.connection_string)
+                    cur.execute("SELECT pg_catalog.citus_set_coordinator_host(%s, %s, 'primary', 'default')",
+                                (r.hostname, r.port or 5432))
         finally:
             conn.close()
 

patroni/postgresql/config.py

@@ -6,7 +6,7 @@ import socket
 import stat
 import time
 
-from six.moves.urllib_parse import urlparse, parse_qsl, unquote
+from urllib.parse import urlparse, parse_qsl, unquote
 
 from .validator import CaseInsensitiveDict, recovery_parameters,\
         transform_postgresql_parameter_value, transform_recovery_parameter_value
@@ -768,9 +768,7 @@ class ConfigHandler(object):
             os.chmod(self._pgpass, stat.S_IWRITE | stat.S_IREAD)
             f.write(line)
 
-        env = os.environ.copy()
+        return {**os.environ, 'PGPASSFILE': self._pgpass}
-        env['PGPASSFILE'] = self._pgpass
-        return env
 
     def write_recovery_conf(self, recovery_params):
         self._recovery_params = recovery_params
@@ -1094,12 +1092,22 @@ class ConfigHandler(object):
                 effective_configuration[name] = cvalue
                 self._postgresql.set_pending_restart(True)
 
-        # If we are using custom bootstrap with PITR it could fail when values
+        # If we are using custom bootstrap with PITR it could fail when values like max_connections
-        # like max_connections are increased, therefore we disable hot_standby.
+        # are increased, therefore we disable hot_standby if recovery_target_action == 'promote'.
-        if self._postgresql.bootstrap.running_custom_bootstrap and \
+        if self._postgresql.bootstrap.running_custom_bootstrap:
-                (self._postgresql.bootstrap.keep_existing_recovery_conf or self._recovery_conf):
+            disable_hot_standby = False
-            effective_configuration['hot_standby'] = 'off'
+            if self._postgresql.bootstrap.keep_existing_recovery_conf:
-            self._postgresql.set_pending_restart(True)
+                disable_hot_standby = True  # trust that pgBackRest does the right thing
+            # `pause_at_recovery_target` has no effect if hot_standby is not enabled, therefore we consider only 9.5+
+            elif self._postgresql.major_version >= 90500 and self._recovery_params:
+                pause_at_recovery_target = parse_bool(self._recovery_params.get('pause_at_recovery_target'))
+                recovery_target_action = self._recovery_params.get(
+                    'recovery_target_action', 'promote' if pause_at_recovery_target is False else 'pause')
+                disable_hot_standby = recovery_target_action == 'promote'
+
+            if disable_hot_standby:
+                effective_configuration['hot_standby'] = 'off'
+                self._postgresql.set_pending_restart(True)
 
         return effective_configuration
 

patroni/postgresql/misc.py

@@ -7,7 +7,7 @@ from patroni.exceptions import PostgresException
 logger = logging.getLogger(__name__)
 
 
-def postgres_version_to_int(pg_version):
+def postgres_version_to_int(pg_version: str) -> int:
     """Convert the server_version to integer
 
     >>> postgres_version_to_int('9.5.3')
@@ -45,7 +45,7 @@ def postgres_version_to_int(pg_version):
     return int(''.join('{0:02d}'.format(c) for c in components))
 
 
-def postgres_major_version_to_int(pg_version):
+def postgres_major_version_to_int(pg_version: str) -> int:
     """
     >>> postgres_major_version_to_int('10')
     100000

patroni/postgresql/rewind.py

@@ -3,7 +3,6 @@ import os
 import re
 import shlex
 import shutil
-import six
 import subprocess
 
 from threading import Lock, Thread
@@ -152,7 +151,7 @@ class Rewind(object):
         else:  # otherwise analyze pg_controldata output
             in_recovery, timeline, lsn = self._get_local_timeline_lsn_from_controldata()
 
-        log_lsn = format_lsn(lsn) if isinstance(lsn, six.integer_types) else lsn
+        log_lsn = format_lsn(lsn) if isinstance(lsn, int) else lsn
         logger.info('Local timeline=%s lsn=%s', timeline, log_lsn)
         return in_recovery, timeline, lsn
 
@@ -215,7 +214,7 @@ class Rewind(object):
                 elif primary_timeline > 1:
                     cur.execute('TIMELINE_HISTORY {0}'.format(primary_timeline))
                     history = cur.fetchone()[1]
-                    if not isinstance(history, six.string_types):
+                    if not isinstance(history, str):
                         history = bytes(history).decode('utf-8')
                     logger.debug('primary: history=%s', history)
         except Exception:

patroni/postgresql/validator.py

@@ -1,6 +1,5 @@
 import abc
 import logging
-import six
 
 from collections import namedtuple
 from urllib3.response import HTTPHeaderDict
@@ -34,8 +33,7 @@ class Bool(namedtuple('Bool', 'version_from,version_till')):
         logger.warning('Removing bool parameter=%s from the config due to the invalid value=%s', name, value)
 
 
-@six.add_metaclass(abc.ABCMeta)
+class Number(abc.ABC, namedtuple('Number', 'version_from,version_till,min_val,max_val,unit')):
-class Number(namedtuple('Number', 'version_from,version_till,min_val,max_val,unit')):
 
     @staticmethod
     @abc.abstractmethod

patroni/request.py

@@ -1,8 +1,7 @@
 import json
 import urllib3
-import six
 
-from six.moves.urllib_parse import urlparse, urlunparse
+from urllib.parse import urlparse, urlunparse
 
 from .utils import USER_AGENT
 
@@ -51,7 +50,7 @@ class PatroniRequest(object):
         self._apply_pool_param('ca_certs', cacert)
 
     def request(self, method, url, body=None, **kwargs):
-        if body is not None and not isinstance(body, six.string_types):
+        if body is not None and not isinstance(body, str):
             body = json.dumps(body)
         return self._pool.request(method.upper(), url, body=body, **kwargs)
 

patroni/scripts/aws.py

@@ -6,9 +6,9 @@ import sys
 import boto3
 
 from ..utils import Retry, RetryFailedError
-from ..request import get as requests_get
 
 from botocore.exceptions import ClientError
+from botocore.utils import IMDSFetcher
 
 logger = logging.getLogger(__name__)
 
@@ -21,14 +21,16 @@ class AWSConnection(object):
         self._retry = Retry(deadline=300, max_delay=30, max_tries=-1, retry_exceptions=(ClientError,))
         try:
             # get the instance id
-            r = requests_get('http://169.254.169.254/latest/dynamic/instance-identity/document', timeout=2.1)
+            fetcher = IMDSFetcher(timeout=2.1)
+            token = fetcher._fetch_metadata_token()
+            r = fetcher._get_request("/latest/dynamic/instance-identity/document", None, token)
         except Exception:
             logger.error('cannot query AWS meta-data')
             return
 
-        if r.status < 400:
+        if r.status_code < 400:
             try:
-                content = json.loads(r.data.decode('utf-8'))
+                content = json.loads(r.text)
                 self.instance_id = content['instanceId']
                 self.region = content['region']
             except Exception:

patroni/utils.py

@@ -514,21 +514,3 @@ def enable_keepalive(sock, timeout, idle, cnt=3):
 
     for opt in keepalive_socket_options(timeout, idle, cnt):
         sock.setsockopt(*opt)
-
-
-def find_executable(executable, path=None):
-    _, ext = os.path.splitext(executable)
-
-    if (sys.platform == 'win32') and (ext == ''):
-        executable = executable + '.exe'  # Set default WIN extension
-
-    if os.path.isfile(executable):
-        return executable
-
-    if path is None:
-        path = os.environ.get('PATH', os.defpath)
-
-    for p in path.split(os.pathsep):
-        f = os.path.join(p, executable)
-        if os.path.isfile(f):
-            return f

patroni/validator.py

@@ -1,12 +1,11 @@
 #!/usr/bin/env python3
 import os
-import socket
 import re
+import shutil
+import socket
 import subprocess
 
-from six import string_types
+from .utils import split_host_port, data_directory_is_empty
-
-from .utils import find_executable, split_host_port, data_directory_is_empty
 from .dcs import dcs_modules
 from .exceptions import ConfigParseError
 
@@ -173,7 +172,7 @@ class Directory(object):
                         yield Result(False, "'{}' does not contain '{}'".format(name, path))
             if self.contains_executable:
                 for program in self.contains_executable:
-                    if not find_executable(program, name):
+                    if not shutil.which(program, path=name):
                         yield Result(False, "'{}' does not contain '{}'".format(name, program))
 
 
@@ -190,12 +189,12 @@ class Schema(object):
 
     def validate(self, data):
         self.data = data
-        if isinstance(self.validator, string_types):
+        if isinstance(self.validator, str):
-            yield Result(isinstance(self.data, string_types), "is not a string", level=1, data=self.data)
+            yield Result(isinstance(self.data, str), "is not a string", level=1, data=self.data)
         elif issubclass(type(self.validator), type):
             validator = self.validator
             if self.validator == str:
-                validator = string_types
+                validator = str
             yield Result(isinstance(self.data, validator),
                          "is not {}".format(_get_type_name(self.validator)), level=1, data=self.data)
         elif callable(self.validator):
@@ -290,8 +289,8 @@ class Schema(object):
 
 
 def _get_type_name(python_type):
-    return {str: 'a string', int: 'and integer', float: 'a number', bool: 'a boolean',
+    return {str: 'a string', int: 'and integer', float: 'a number',
-            list: 'an array', dict: 'a dictionary', string_types: "a string"}.get(
+            bool: 'a boolean', list: 'an array', dict: 'a dictionary'}.get(
                     python_type, getattr(python_type, __name__, "unknown type"))
 
 
@@ -302,11 +301,11 @@ def assert_(condition, message="Wrong value"):
 userattributes = {"username": "", Optional("password"): ""}
 available_dcs = [m.split(".")[-1] for m in dcs_modules()]
 validate_host_port_list.expected_type = list
-comma_separated_host_port.expected_type = string_types
+comma_separated_host_port.expected_type = str
-validate_connect_address.expected_type = string_types
+validate_connect_address.expected_type = str
-validate_host_port_listen.expected_type = string_types
+validate_host_port_listen.expected_type = str
-validate_host_port_listen_multiple_hosts.expected_type = string_types
+validate_host_port_listen_multiple_hosts.expected_type = str
-validate_data_dir.expected_type = string_types
+validate_data_dir.expected_type = str
 validate_etcd = {
     Or("host", "hosts", "srv", "srv_suffix", "url", "proxy"): Case({
         "host": validate_host_port,
@@ -365,6 +364,7 @@ schema = Schema({
           Optional("use_endpoints"): bool,
           Optional("pod_ip"): Or(is_ipv4_address, is_ipv6_address),
           Optional("ports"): [{"name": str, "port": int}],
+          Optional("retriable_http_codes"): Or(int, [int]),
           },
       }),
   Optional("citus"): {
@@ -384,7 +384,7 @@ schema = Schema({
     Optional("bin_dir"): Directory(contains_executable=["pg_ctl", "initdb", "pg_controldata", "pg_basebackup",
                                                         "postgres", "pg_isready"]),
     Optional("parameters"): {
-      Optional("unix_socket_directories"): lambda s: assert_(all([isinstance(s, string_types), len(s)]))
+      Optional("unix_socket_directories"): lambda s: assert_(all([isinstance(s, str), len(s)]))
     },
     Optional("pg_hba"): [str],
     Optional("pg_ident"): [str],

patroni/version.py

@@ -1 +1,5 @@
-__version__ = '3.0.0'
+"""This module specifies the current Patroni version.
+
+:var __version__: the current Patroni version.
+"""
+__version__ = '3.0.2'

patroni/watchdog/base.py

@@ -1,7 +1,6 @@
 import abc
 import logging
 import platform
-import six
 import sys
 from threading import RLock
 
@@ -235,8 +234,7 @@ class Watchdog(object):
         return self.config.timing_slack >= 0 and self.impl.is_healthy
 
 
-@six.add_metaclass(abc.ABCMeta)
+class WatchdogBase(abc.ABC):
-class WatchdogBase(object):
     """A watchdog object when opened requires periodic calls to keepalive.
     When keepalive is not called within a timeout the system will be terminated."""
     is_null = False

requirements.txt

@@ -1,8 +1,6 @@
 urllib3>=1.19.1,!=1.21
-ipaddress; python_version=="2.7"
 boto3
 PyYAML
-six >= 1.7
 kazoo>=1.3.1
 python-etcd>=0.4.3,<0.5
 python-consul>=0.7.1

setup.py

@@ -42,6 +42,7 @@ CLASSIFIERS = [
     'Operating System :: Microsoft :: Windows',
     'Programming Language :: Python',
     'Programming Language :: Python :: 3',
+    'Programming Language :: Python :: 3.6',
     'Programming Language :: Python :: 3.7',
     'Programming Language :: Python :: 3.8',
     'Programming Language :: Python :: 3.9',
@@ -93,7 +94,7 @@ class Flake8(_Command):
         from flake8.main.cli import main
 
         logging.getLogger().setLevel(logging.ERROR)
-        main(self.targets())
+        raise SystemExit(main(self.targets()))
 
 
 class PyTest(_Command):

tests/test_api.py

@@ -5,14 +5,16 @@ import socket
 
 import patroni.psycopg as psycopg
 
+from http.server import HTTPServer
+from io import BytesIO as IO
 from mock import Mock, PropertyMock, patch
+from socketserver import ThreadingMixIn
+
 from patroni.api import RestApiHandler, RestApiServer
 from patroni.dcs import ClusterConfig, Member
 from patroni.ha import _MemberStatus
 from patroni.utils import tzutc
-from six import BytesIO as IO
+
-from six.moves import BaseHTTPServer
-from six.moves.socketserver import ThreadingMixIn
 from . import psycopg_connect, MockCursor
 from .test_ha import get_cluster_initialized_without_leader
 
@@ -175,13 +177,14 @@ class MockRestApiServer(RestApiServer):
 
 @patch('ssl.SSLContext.load_cert_chain', Mock())
 @patch('ssl.SSLContext.wrap_socket', Mock(return_value=0))
-@patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
+@patch.object(HTTPServer, '__init__', Mock())
 class TestRestApiHandler(unittest.TestCase):
 
     _authorization = '\nAuthorization: Basic dGVzdDp0ZXN0'
 
     def test_do_GET(self):
         MockPatroni.dcs.cluster.last_lsn = 20
+        MockPatroni.dcs.cluster.sync.members = [MockPostgresql.name]
         MockRestApiServer(RestApiHandler, 'GET /replica')
         MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M')
         MockRestApiServer(RestApiHandler, 'GET /replica?lag=10MB')
@@ -194,9 +197,8 @@ class TestRestApiHandler(unittest.TestCase):
         with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'state': 'running'})):
             MockRestApiServer(RestApiHandler, 'GET /health')
         MockRestApiServer(RestApiHandler, 'GET /leader')
-        MockPatroni.dcs.cluster.sync.members = [MockPostgresql.name]
+        with patch.object(RestApiHandler, 'get_postgresql_status',
-        MockPatroni.dcs.cluster.is_synchronous_mode = Mock(return_value=True)
+                          Mock(return_value={'role': 'replica', 'sync_standby': True})):
-        with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'replica'})):
             MockRestApiServer(RestApiHandler, 'GET /synchronous')
             MockRestApiServer(RestApiHandler, 'GET /read-only-sync')
         with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'replica'})):
@@ -587,14 +589,14 @@ class TestRestApiServer(unittest.TestCase):
     @patch('ssl.SSLContext.load_cert_chain', Mock())
     @patch('ssl.SSLContext.set_ciphers', Mock())
     @patch('ssl.SSLContext.wrap_socket', Mock(return_value=0))
-    @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
+    @patch.object(HTTPServer, '__init__', Mock())
     def setUp(self):
         self.srv = MockRestApiServer(Mock(), '', {'listen': '*:8008', 'certfile': 'a', 'verify_client': 'required',
                                                   'ciphers': '!SSLv1:!SSLv2:!SSLv3:!TLSv1:!TLSv1.1',
                                                   'allowlist': ['127.0.0.1', '::1/128', '::1/zxc'],
                                                   'allowlist_include_members': True})
 
-    @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
+    @patch.object(HTTPServer, '__init__', Mock())
     def test_reload_config(self):
         bad_config = {'listen': 'foo'}
         self.assertRaises(ValueError, MockRestApiServer, None, '', bad_config)
@@ -622,7 +624,7 @@ class TestRestApiServer(unittest.TestCase):
         except Exception:
             self.assertIsNone(MockRestApiServer.handle_error(None, ('127.0.0.1', 55555)))
 
-    @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock(side_effect=socket.error))
+    @patch.object(HTTPServer, '__init__', Mock(side_effect=socket.error))
     def test_socket_error(self):
         self.assertRaises(socket.error, MockRestApiServer, Mock(), '', {'listen': '*:8008'})
 

tests/test_aws.py

@@ -1,9 +1,9 @@
 import botocore
+import botocore.awsrequest
 import sys
 import unittest
-import urllib3
 
-from mock import Mock, patch
+from mock import Mock, PropertyMock, patch
 from collections import namedtuple
 from patroni.scripts.aws import AWSConnection, main as _main
 
@@ -28,31 +28,45 @@ class MockEc2Connection(object):
         return True
 
 
+class MockIMDSFetcher(object):
+
+    def __init__(self, timeout):
+        pass
+
+    @staticmethod
+    def _fetch_metadata_token():
+        return ''
+
+    @staticmethod
+    def _get_request(*args):
+        return botocore.awsrequest.AWSResponse(url='', status_code=200, headers={}, raw=None)
+
+
 @patch('boto3.resource', Mock(return_value=MockEc2Connection()))
+@patch('patroni.scripts.aws.IMDSFetcher', MockIMDSFetcher)
 class TestAWSConnection(unittest.TestCase):
 
-    @patch('patroni.scripts.aws.requests_get', Mock(return_value=urllib3.HTTPResponse(
+    @patch.object(botocore.awsrequest.AWSResponse, 'text',
-        status=200, body=b'{"instanceId": "012345", "region": "eu-west-1"}')))
+                  PropertyMock(return_value='{"instanceId": "012345", "region": "eu-west-1"}'))
-    def setUp(self):
-        self.conn = AWSConnection('test')
-
     def test_on_role_change(self):
-        self.assertTrue(self.conn.on_role_change('primary'))
+        conn = AWSConnection('test')
+        self.assertTrue(conn.on_role_change('primary'))
         with patch.object(MockVolumes, 'filter', Mock(return_value=[])):
-            self.conn._retry.max_tries = 1
+            conn._retry.max_tries = 1
-            self.assertFalse(self.conn.on_role_change('primary'))
+            self.assertFalse(conn.on_role_change('primary'))
 
-    @patch('patroni.scripts.aws.requests_get', Mock(side_effect=Exception('foo')))
+    @patch.object(MockIMDSFetcher, '_get_request', Mock(side_effect=Exception('foo')))
     def test_non_aws(self):
         conn = AWSConnection('test')
         self.assertFalse(conn.on_role_change("primary"))
 
-    @patch('patroni.scripts.aws.requests_get', Mock(return_value=urllib3.HTTPResponse(status=200, body=b'foo')))
+    @patch.object(botocore.awsrequest.AWSResponse, 'text', PropertyMock(return_value='boo'))
     def test_aws_bizare_response(self):
         conn = AWSConnection('test')
         self.assertFalse(conn.aws_available())
 
-    @patch('patroni.scripts.aws.requests_get', Mock(return_value=urllib3.HTTPResponse(status=503, body=b'Error')))
+    @patch.object(MockIMDSFetcher, '_get_request', Mock(return_value=botocore.awsrequest.AWSResponse(
+        url='', status_code=503, headers={}, raw=None)))
     @patch('sys.exit', Mock())
     def test_main(self):
         self.assertIsNone(_main())

tests/test_bootstrap.py

@@ -112,6 +112,7 @@ class TestBootstrap(BaseTestPostgresql):
         config = {'users': {'replicator': {'password': 'rep-pass', 'options': ['replication']}}}
 
         with patch.object(Postgresql, 'is_running', Mock(return_value=False)),\
+                patch.object(Postgresql, 'get_major_version', Mock(return_value=140000)),\
                 patch('multiprocessing.Process', Mock(side_effect=Exception)),\
                 patch('multiprocessing.get_context', Mock(side_effect=Exception), create=True):
             self.assertRaises(Exception, self.b.bootstrap, config)

tests/test_callback_executor.py

@@ -12,25 +12,28 @@ class TestCallbackExecutor(unittest.TestCase):
         mock_popen.return_value.children.return_value = []
         mock_popen.return_value.is_running.return_value = True
 
+        callback = ['test.sh', 'on_start', 'replica', 'foo']
         ce = CallbackExecutor()
         ce._kill_children = Mock(side_effect=Exception)
         ce._invoke_excepthook = Mock()
-        self.assertIsNone(ce.call([]))
+        self.assertIsNone(ce.call(callback))
         ce.join()
 
-        self.assertIsNone(ce.call([]))
+        self.assertIsNone(ce.call(callback))
 
         mock_popen.return_value.kill.side_effect = psutil.AccessDenied()
-        self.assertIsNone(ce.call([]))
+        self.assertIsNone(ce.call(callback))
 
         ce._process_children = []
         mock_popen.return_value.children.side_effect = psutil.Error()
         mock_popen.return_value.kill.side_effect = psutil.NoSuchProcess(123)
-        self.assertIsNone(ce.call([]))
+        self.assertIsNone(ce.call(callback))
 
         mock_popen.side_effect = Exception
         ce = CallbackExecutor()
         ce._condition.wait = Mock(side_effect=[None, Exception])
         ce._invoke_excepthook = Mock()
-        self.assertIsNone(ce.call([]))
+        self.assertIsNone(ce.call(callback))
+
+        self.assertIsNone(ce.call(['test.sh', 'on_reload', 'replica', 'foo']))
         ce.join()

tests/test_config.py

@@ -5,14 +5,13 @@ import io
 
 from mock import MagicMock, Mock, patch
 from patroni.config import Config, ConfigParseError
-from six.moves import builtins
 
 
 class TestConfig(unittest.TestCase):
 
     @patch('os.path.isfile', Mock(return_value=True))
     @patch('json.load', Mock(side_effect=Exception))
-    @patch.object(builtins, 'open', MagicMock())
+    @patch('builtins.open', MagicMock())
     def setUp(self):
         sys.argv = ['patroni.py']
         os.environ[Config.PATRONI_CONFIG_VARIABLE] = 'restapi: {}\npostgresql: {data_dir: foo}'
@@ -61,6 +60,7 @@ class TestConfig(unittest.TestCase):
             'PATRONI_KUBERNETES_LABELS': 'a: b: c',
             'PATRONI_KUBERNETES_SCOPE_LABEL': 'a',
             'PATRONI_KUBERNETES_PORTS': '[{"name": "postgresql"}]',
+            'PATRONI_KUBERNETES_RETRIABLE_HTTP_CODES': '401',
             'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
             'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',
             'PATRONI_EXHIBITOR_PORT': '8181',
@@ -136,7 +136,7 @@ class TestConfig(unittest.TestCase):
                     new-attr: True
                     ''')
 
-        with patch.object(builtins, 'open', MagicMock(side_effect=open_mock)):
+        with patch('builtins.open', MagicMock(side_effect=open_mock)):
             config = Config('postgres0')
             self.assertEqual(config._local_configuration,
                              {'test': False, 'test2': {'child-1': 'somestring', 'child-2': 10},

tests/test_ctl.py

@@ -4,7 +4,7 @@ import unittest
 
 from click.testing import CliRunner
 from datetime import datetime, timedelta
-from mock import patch, Mock
+from mock import patch, Mock, call
 from patroni.ctl import ctl, load_config, output_members, get_dcs, parse_dcs, \
     get_all_members, get_any_member, get_cursor, query_member, PatroniCtlException, apply_config_changes, \
     format_config_for_editing, show_diff, invoke_editor, format_pg_version, CONFIG_FILE_PATH, PatronictlPrettyTable
@@ -28,12 +28,11 @@ class TestCtl(unittest.TestCase):
     TEST_ROLES = ('master', 'primary', 'leader')
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
+    @patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379']))
     def setUp(self):
-        with patch.object(AbstractEtcdClientWithFailover, 'machines') as mock_machines:
+        self.runner = CliRunner()
-            mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
+        self.e = get_dcs({'etcd': {'ttl': 30, 'host': 'ok:2379', 'retry_timeout': 10},
-            self.runner = CliRunner()
+                          'citus': {'group': 0}}, 'foo', None)
-            self.e = get_dcs({'etcd': {'ttl': 30, 'host': 'ok:2379', 'retry_timeout': 10},
-                              'citus': {'group': 0}}, 'foo', None)
 
     @patch('patroni.ctl.logging.debug')
     def test_load_config(self, mock_logger_debug):
@@ -559,19 +558,55 @@ class TestCtl(unittest.TestCase):
 
     @patch('sys.stdout.isatty', return_value=False)
     @patch('patroni.ctl.markup_to_pager')
-    @patch('patroni.ctl.find_executable', return_value=None)
+    @patch('os.environ.get', return_value=None)
-    def test_show_diff(self, mock_find_executable, mock_markup_to_pager, mock_isatty):
+    @patch('shutil.which', return_value=None)
+    def test_show_diff(self, mock_which, mock_env_get, mock_markup_to_pager, mock_isatty):
+        # no TTY
         show_diff("foo:\n  bar: 1\n", "foo:\n  bar: 2\n")
         mock_markup_to_pager.assert_not_called()
 
+        # TTY but no PAGER nor executable
         mock_isatty.return_value = True
+        with self.assertRaises(PatroniCtlException) as e:
+            show_diff("foo:\n  bar: 1\n", "foo:\n  bar: 2\n")
+        self.assertEqual(
+            str(e.exception),
+            'No pager could be found. Either set PAGER environment variable with '
+            'your pager or install either "less" or "more" in the host.'
+        )
+        mock_env_get.assert_called_once_with('PAGER')
+        mock_which.assert_has_calls([
+            call('less'),
+            call('more'),
+        ])
+        mock_markup_to_pager.assert_not_called()
+
+        # TTY with PAGER set but invalid
+        mock_env_get.reset_mock()
+        mock_env_get.return_value = 'random'
+        mock_which.reset_mock()
+        with self.assertRaises(PatroniCtlException) as e:
+            show_diff("foo:\n  bar: 1\n", "foo:\n  bar: 2\n")
+        self.assertEqual(
+            str(e.exception),
+            'No pager could be found. Either set PAGER environment variable with '
+            'your pager or install either "less" or "more" in the host.'
+        )
+        mock_env_get.assert_called_once_with('PAGER')
+        mock_which.assert_has_calls([
+            call('random'),
+            call('less'),
+            call('more'),
+        ])
+        mock_markup_to_pager.assert_not_called()
+
+        # TTY with valid executable
+        mock_which.side_effect = [None, '/usr/bin/less', None]
         show_diff("foo:\n  bar: 1\n", "foo:\n  bar: 2\n")
         mock_markup_to_pager.assert_called_once()
 
-        show_diff("foo:\n  bar: 1\n", "foo:\n  bar: 2\n")
-
         # Test that unicode handling doesn't fail with an exception
-        mock_find_executable.return_value = '/usr/bin/less'
+        mock_which.side_effect = [None, '/usr/bin/less', None]
         show_diff(b"foo:\n  bar: \xc3\xb6\xc3\xb6\n".decode('utf-8'),
                   b"foo:\n  bar: \xc3\xbc\xc3\xbc\n".decode('utf-8'))
 
@@ -579,7 +614,7 @@ class TestCtl(unittest.TestCase):
     def test_invoke_editor(self, mock_subprocess_call):
         os.environ.pop('EDITOR', None)
         for e in ('', '/bin/vi'):
-            with patch('patroni.ctl.find_executable', Mock(return_value=e)):
+            with patch('shutil.which', Mock(return_value=e)):
                 self.assertRaises(PatroniCtlException, invoke_editor, 'foo: bar\n', 'test')
 
     @patch('patroni.ctl.get_dcs')

tests/test_etcd.py

@@ -135,12 +135,12 @@ class TestClient(unittest.TestCase):
     @patch('dns.resolver.query', dns_query)
     @patch('socket.getaddrinfo', socket_getaddrinfo)
     @patch('patroni.dcs.etcd.requests_get', requests_get)
+    @patch.object(EtcdClient, '_get_machines_list',
+                  Mock(return_value=['http://localhost:2379', 'http://localhost:4001']))
     def setUp(self):
-        with patch.object(EtcdClient, 'machines') as mock_machines:
+        self.client = EtcdClient({'srv': 'test', 'retry_timeout': 3}, DnsCachingResolver())
-            mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
+        self.client.http.request = http_request
-            self.client = EtcdClient({'srv': 'test', 'retry_timeout': 3}, DnsCachingResolver())
+        self.client.http.request_encode_body = http_request
-            self.client.http.request = http_request
-            self.client.http.request_encode_body = http_request
 
     def test_machines(self):
         self.client._base_uri = 'http://localhost:4002'
@@ -156,9 +156,9 @@ class TestClient(unittest.TestCase):
         except Exception:
             self.assertIsNone(machines)
 
-    @patch.object(EtcdClient, 'machines')
+    @patch.object(EtcdClient, '_get_machines_list',
-    def test_api_execute(self, mock_machines):
+                  Mock(return_value=['http://localhost:4001', 'http://localhost:2379']))
-        mock_machines.__get__ = Mock(return_value=['http://localhost:4001', 'http://localhost:2379'])
+    def test_api_execute(self):
         self.client._base_uri = 'http://localhost:4001'
         self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'POST', timeout=0)
         self.client._base_uri = 'http://localhost:4001'
@@ -196,11 +196,10 @@ class TestClient(unittest.TestCase):
     def test__get_machines_cache_from_dns(self):
         self.client._get_machines_cache_from_dns('error', 2379)
 
-    @patch.object(EtcdClient, 'machines')
+    @patch.object(EtcdClient, '_get_machines_list', Mock(side_effect=etcd.EtcdConnectionFailed))
-    def test__refresh_machines_cache(self, mock_machines):
+    def test__refresh_machines_cache(self):
-        mock_machines.__get__ = Mock(side_effect=etcd.EtcdConnectionFailed)
+        self.assertFalse(self.client._refresh_machines_cache())
-        self.assertIsNone(self.client._refresh_machines_cache())
+        self.assertRaises(etcd.EtcdException, self.client._refresh_machines_cache, ['http://localhost:2379'])
-        self.assertRaises(etcd.EtcdException, self.client._refresh_machines_cache, True)
 
     def test__load_machines_cache(self):
         self.client._config = {}
@@ -230,28 +229,27 @@ class TestClient(unittest.TestCase):
 class TestEtcd(unittest.TestCase):
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
+    @patch.object(EtcdClient, '_get_machines_list',
+                  Mock(return_value=['http://localhost:2379', 'http://localhost:4001']))
     def setUp(self):
-        with patch.object(EtcdClient, 'machines') as mock_machines:
+        self.etcd = Etcd({'namespace': '/patroni/', 'ttl': 30, 'retry_timeout': 10,
-            mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
+                          'host': 'localhost:2379', 'scope': 'test', 'name': 'foo'})
-            self.etcd = Etcd({'namespace': '/patroni/', 'ttl': 30, 'retry_timeout': 10,
-                              'host': 'localhost:2379', 'scope': 'test', 'name': 'foo'})
 
     def test_base_path(self):
         self.assertEqual(self.etcd._base_path, '/patroni/test')
 
     @patch('dns.resolver.query', dns_query)
+    @patch('time.sleep', Mock(side_effect=SleepException))
+    @patch.object(EtcdClient, '_get_machines_list', Mock(side_effect=etcd.EtcdConnectionFailed))
     def test_get_etcd_client(self):
-        with patch('time.sleep', Mock(side_effect=SleepException)),\
+        self.assertRaises(SleepException, self.etcd.get_etcd_client,
-                patch.object(EtcdClient, 'machines') as mock_machines:
+                          {'discovery_srv': 'test', 'retry_timeout': 10, 'cacert': '1', 'key': '1', 'cert': 1},
-            mock_machines.__get__ = Mock(side_effect=etcd.EtcdException)
+                          EtcdClient)
-            self.assertRaises(SleepException, self.etcd.get_etcd_client,
+        self.assertRaises(SleepException, self.etcd.get_etcd_client,
-                              {'discovery_srv': 'test', 'retry_timeout': 10, 'cacert': '1', 'key': '1', 'cert': 1},
+                          {'url': 'https://test:2379', 'retry_timeout': 10}, EtcdClient)
-                              EtcdClient)
+        self.assertRaises(SleepException, self.etcd.get_etcd_client,
-            self.assertRaises(SleepException, self.etcd.get_etcd_client,
+                          {'hosts': 'foo:4001,bar', 'retry_timeout': 10}, EtcdClient)
-                              {'url': 'https://test:2379', 'retry_timeout': 10}, EtcdClient)
+        with patch.object(EtcdClient, '_get_machines_list', Mock(return_value=[])):
-            self.assertRaises(SleepException, self.etcd.get_etcd_client,
-                              {'hosts': 'foo:4001,bar', 'retry_timeout': 10}, EtcdClient)
-            mock_machines.__get__ = Mock(return_value=[])
             self.assertRaises(SleepException, self.etcd.get_etcd_client,
                               {'proxy': 'https://user:password@test:2379', 'retry_timeout': 10}, EtcdClient)
 

tests/test_ha.py

@@ -18,7 +18,6 @@ from patroni.postgresql.rewind import Rewind
 from patroni.postgresql.slots import SlotsHandler
 from patroni.utils import tzutc
 from patroni.watchdog import Watchdog
-from six.moves import builtins
 
 from . import PostgresInit, MockPostmaster, psycopg_connect, requests_get
 from .test_etcd import socket_getaddrinfo, etcd_read, etcd_write
@@ -43,11 +42,11 @@ def get_cluster(initialize, leader, members, failover, sync, cluster_config=None
 
 
 def get_cluster_not_initialized_without_leader(cluster_config=None):
-    return get_cluster(None, None, [], None, SyncState(None, None, None), cluster_config)
+    return get_cluster(None, None, [], None, SyncState.empty(), cluster_config)
 
 
 def get_cluster_bootstrapping_without_leader(cluster_config=None):
-    return get_cluster("", None, [], None, SyncState(None, None, None), cluster_config)
+    return get_cluster("", None, [], None, SyncState.empty(), cluster_config)
 
 
 def get_cluster_initialized_without_leader(leader=False, failover=None, sync=None, cluster_config=None, failsafe=False):
@@ -73,7 +72,7 @@ def get_cluster_initialized_with_leader(failover=None, sync=None):
 
 def get_cluster_initialized_with_only_leader(failover=None, cluster_config=None):
     leader = get_cluster_initialized_without_leader(leader=True, failover=failover).leader
-    return get_cluster(True, leader, [leader.member], failover, None, cluster_config)
+    return get_cluster(True, leader, [leader.member], failover, SyncState.empty(), cluster_config)
 
 
 def get_standby_cluster_initialized_with_only_leader(failover=None, sync=None):
@@ -182,6 +181,8 @@ def run_async(self, func, args=()):
 @patch.object(CancellableSubprocess, 'call', Mock(return_value=0))
 @patch.object(Postgresql, 'get_replica_timeline', Mock(return_value=2))
 @patch.object(Postgresql, 'get_primary_timeline', Mock(return_value=2))
+@patch.object(Postgresql, 'get_major_version', Mock(return_value=140000))
+@patch.object(Postgresql, 'resume_wal_replay', Mock())
 @patch.object(ConfigHandler, 'restore_configuration_files', Mock())
 @patch.object(etcd.Client, 'write', etcd_write)
 @patch.object(etcd.Client, 'read', etcd_read)
@@ -198,21 +199,20 @@ class TestHa(PostgresInit):
     @patch('socket.getaddrinfo', socket_getaddrinfo)
     @patch('patroni.dcs.dcs_modules', Mock(return_value=['patroni.dcs.etcd']))
     @patch.object(etcd.Client, 'read', etcd_read)
+    @patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379']))
     def setUp(self):
         super(TestHa, self).setUp()
-        with patch.object(AbstractEtcdClientWithFailover, 'machines') as mock_machines:
+        self.p.set_state('running')
-            mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
+        self.p.set_role('replica')
-            self.p.set_state('running')
+        self.p.postmaster_start_time = MagicMock(return_value=str(postmaster_start_time))
-            self.p.set_role('replica')
+        self.p.can_create_replica_without_replication_connection = MagicMock(return_value=False)
-            self.p.postmaster_start_time = MagicMock(return_value=str(postmaster_start_time))
+        self.e = get_dcs({'etcd': {'ttl': 30, 'host': 'ok:2379', 'scope': 'test',
-            self.p.can_create_replica_without_replication_connection = MagicMock(return_value=False)
+                                   'name': 'foo', 'retry_timeout': 10},
-            self.e = get_dcs({'etcd': {'ttl': 30, 'host': 'ok:2379', 'scope': 'test',
+                          'citus': {'database': 'citus', 'group': None}})
-                                       'name': 'foo', 'retry_timeout': 10},
+        self.ha = Ha(MockPatroni(self.p, self.e))
-                              'citus': {'database': 'citus', 'group': None}})
+        self.ha.old_cluster = self.e.get_cluster()
-            self.ha = Ha(MockPatroni(self.p, self.e))
+        self.ha.cluster = get_cluster_initialized_without_leader()
-            self.ha.old_cluster = self.e.get_cluster()
+        self.ha.load_cluster_from_dcs = Mock()
-            self.ha.cluster = get_cluster_initialized_without_leader()
-            self.ha.load_cluster_from_dcs = Mock()
 
     def test_update_lock(self):
         self.ha.is_failsafe_mode = true
@@ -449,7 +449,6 @@ class TestHa(PostgresInit):
         self.p.is_leader = false
         self.assertEqual(self.ha.run_cycle(), 'not promoting because failed to update leader lock in DCS')
 
-    @patch.object(Postgresql, 'major_version', PropertyMock(return_value=130000))
     def test_follow(self):
         self.ha.cluster.is_unlocked = false
         self.p.is_leader = false
@@ -941,7 +940,7 @@ class TestHa(PostgresInit):
         self.assertEqual(self.ha.run_cycle(), 'PAUSE: waiting to become primary after promote...')
 
     @patch('patroni.postgresql.mtime', Mock(return_value=1588316884))
-    @patch.object(builtins, 'open', mock_open(read_data='1\t0/40159C0\tno recovery target specified\n'))
+    @patch('builtins.open', mock_open(read_data='1\t0/40159C0\tno recovery target specified\n'))
     def test_process_healthy_standby_cluster_as_standby_leader(self):
         self.p.is_leader = false
         self.p.name = 'leader'
@@ -1273,7 +1272,7 @@ class TestHa(PostgresInit):
         self.assertEqual(self.ha.get_effective_tags(), {'foo': 'bar'})
 
     @patch('patroni.postgresql.mtime', Mock(return_value=1588316884))
-    @patch.object(builtins, 'open', Mock(side_effect=Exception))
+    @patch('builtins.open', Mock(side_effect=Exception))
     def test_restore_cluster_config(self):
         self.ha.cluster.config.data.clear()
         self.ha.has_lock = true
@@ -1326,8 +1325,8 @@ class TestHa(PostgresInit):
                          "data directory is not accessible: [Errno 5] Input/output error: '{}'".format(self.p.data_dir))
 
     @patch('patroni.postgresql.mtime', Mock(return_value=1588316884))
-    @patch.object(builtins, 'open', mock_open(read_data=('1\t0/40159C0\tno recovery target specified\n\n'
+    @patch('builtins.open', mock_open(read_data=('1\t0/40159C0\tno recovery target specified\n\n'
-                                                         '2\t1/40159C0\tno recovery target specified\n')))
+                                                 '2\t1/40159C0\tno recovery target specified\n')))
     def test_update_cluster_history(self):
         self.ha.has_lock = true
         self.ha.cluster.is_unlocked = false
@@ -1391,7 +1390,7 @@ class TestHa(PostgresInit):
     @patch('os.close', Mock())
     @patch('os.rename', Mock())
     @patch('patroni.postgresql.Postgresql.is_starting', Mock(return_value=False))
-    @patch.object(builtins, 'open', mock_open())
+    @patch('builtins.open', mock_open())
     @patch.object(ConfigHandler, 'check_recovery_conf', Mock(return_value=(False, False)))
     @patch.object(Postgresql, 'major_version', PropertyMock(return_value=130000))
     @patch.object(SlotsHandler, 'sync_replication_slots', Mock(return_value=['ls']))

tests/test_kubernetes.py

@@ -10,7 +10,6 @@ from mock import Mock, PropertyMock, mock_open, patch
 from patroni.dcs.kubernetes import Cluster, k8s_client, k8s_config, K8sConfig, K8sConnectionFailed,\
         K8sException, K8sObject, Kubernetes, KubernetesError, KubernetesRetriableException,\
         Retry, RetryFailedError, SERVICE_HOST_ENV_NAME, SERVICE_PORT_ENV_NAME
-from six.moves import builtins
 from threading import Thread
 from . import MockResponse, SleepException
 
@@ -85,7 +84,7 @@ class TestK8sConfig(unittest.TestCase):
 
         with patch('os.environ', {SERVICE_HOST_ENV_NAME: 'a', SERVICE_PORT_ENV_NAME: '1'}),\
                 patch('os.path.isfile', Mock(side_effect=[False, True, True, False, True, True, True, True])),\
-                patch.object(builtins, 'open', Mock(side_effect=[
+                patch('builtins.open', Mock(side_effect=[
                     mock_open()(), mock_open(read_data='a')(), mock_open(read_data='a')(),
                     mock_open()(), mock_open(read_data='a')(), mock_open(read_data='a')()])):
             for _ in range(0, 4):
@@ -97,7 +96,7 @@ class TestK8sConfig(unittest.TestCase):
     def test_refresh_token(self):
         with patch('os.environ', {SERVICE_HOST_ENV_NAME: 'a', SERVICE_PORT_ENV_NAME: '1'}),\
                 patch('os.path.isfile', Mock(side_effect=[True, True, False, True, True, True])),\
-                patch.object(builtins, 'open', Mock(side_effect=[
+                patch('builtins.open', Mock(side_effect=[
                     mock_open(read_data='cert')(), mock_open(read_data='a')(),
                     mock_open()(), mock_open(read_data='b')(), mock_open(read_data='c')()])):
             k8s_config.load_incluster_config(token_refresh_interval=datetime.timedelta(milliseconds=100))
@@ -122,20 +121,20 @@ class TestK8sConfig(unittest.TestCase):
             "clusters": [{"name": "local", "cluster": {"server": "https://a:1/", "certificate-authority": "a"}}],
             "users": [{"name": "local", "user": {"username": "a", "password": "b", "client-certificate": "c"}}]
         }
-        with patch.object(builtins, 'open', mock_open(read_data=json.dumps(config))):
+        with patch('builtins.open', mock_open(read_data=json.dumps(config))):
             k8s_config.load_kube_config()
             self.assertEqual(k8s_config.server, 'https://a:1')
             self.assertEqual(k8s_config.pool_config, {'ca_certs': 'a', 'cert_file': 'c', 'cert_reqs': 'CERT_REQUIRED',
                                                       'maxsize': 10, 'num_pools': 10})
 
         config["users"][0]["user"]["token"] = "token"
-        with patch.object(builtins, 'open', mock_open(read_data=json.dumps(config))):
+        with patch('builtins.open', mock_open(read_data=json.dumps(config))):
             k8s_config.load_kube_config()
             self.assertEqual(k8s_config.headers.get('authorization'), 'Bearer token')
 
         config["users"][0]["user"]["client-key-data"] = base64.b64encode(b'foobar').decode('utf-8')
         config["clusters"][0]["cluster"]["certificate-authority-data"] = base64.b64encode(b'foobar').decode('utf-8')
-        with patch.object(builtins, 'open', mock_open(read_data=json.dumps(config))),\
+        with patch('builtins.open', mock_open(read_data=json.dumps(config))),\
                 patch('os.write', Mock()), patch('os.close', Mock()),\
                 patch('os.remove') as mock_remove,\
                 patch('atexit.register') as mock_atexit,\
@@ -317,6 +316,17 @@ class TestKubernetesConfigMaps(BaseTestKubernetes):
     def test_set_history_value(self):
         self.k.set_history_value('{}')
 
+    @patch('patroni.dcs.kubernetes.logger.warning')
+    def test_reload_config(self, mock_warning):
+        self.k.reload_config({'loop_wait': 10, 'ttl': 30, 'retry_timeout': 10, 'retriable_http_codes': '401, 403 '})
+        self.assertEqual(self.k._api._retriable_http_codes, self.k._api._DEFAULT_RETRIABLE_HTTP_CODES | set([401, 403]))
+        self.k.reload_config({'loop_wait': 10, 'ttl': 30, 'retry_timeout': 10, 'retriable_http_codes': 402})
+        self.assertEqual(self.k._api._retriable_http_codes, self.k._api._DEFAULT_RETRIABLE_HTTP_CODES | set([402]))
+        self.k.reload_config({'loop_wait': 10, 'ttl': 30, 'retry_timeout': 10, 'retriable_http_codes': [405, 406]})
+        self.assertEqual(self.k._api._retriable_http_codes, self.k._api._DEFAULT_RETRIABLE_HTTP_CODES | set([405, 406]))
+        self.k.reload_config({'loop_wait': 10, 'ttl': 30, 'retry_timeout': 10, 'retriable_http_codes': True})
+        mock_warning.assert_called_once()
+
 
 class TestKubernetesEndpoints(BaseTestKubernetes):
 

tests/test_log.py

@@ -7,7 +7,7 @@ import yaml
 from mock import Mock, patch
 from patroni.config import Config
 from patroni.log import PatroniLogger
-from six.moves.queue import Queue, Full
+from queue import Queue, Full
 
 _LOG = logging.getLogger(__name__)
 

tests/test_patroni.py

@@ -6,6 +6,7 @@ import time
 import unittest
 
 import patroni.config as config
+from http.server import HTTPServer
 from mock import Mock, PropertyMock, patch
 from patroni.api import RestApiServer
 from patroni.async_executor import AsyncExecutor
@@ -15,7 +16,6 @@ from patroni.postgresql import Postgresql
 from patroni.postgresql.config import ConfigHandler
 from patroni import check_psycopg
 from patroni.__main__ import Patroni, main as _main, patroni_main
-from six.moves import BaseHTTPServer, builtins
 from threading import Thread
 
 from . import psycopg_connect, SleepException
@@ -44,7 +44,7 @@ class MockFrozenImporter(object):
 @patch.object(ConfigHandler, 'write_recovery_conf', Mock())
 @patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
 @patch.object(Postgresql, 'call_nowait', Mock())
-@patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
+@patch.object(HTTPServer, '__init__', Mock())
 @patch.object(AsyncExecutor, 'run', Mock())
 @patch.object(etcd.Client, 'write', etcd_write)
 @patch.object(etcd.Client, 'read', etcd_read)
@@ -63,10 +63,10 @@ class TestPatroni(unittest.TestCase):
 
     @patch('pkgutil.iter_importers', Mock(return_value=[MockFrozenImporter()]))
     @patch('sys.frozen', Mock(return_value=True), create=True)
-    @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
+    @patch.object(HTTPServer, '__init__', Mock())
     @patch.object(etcd.Client, 'read', etcd_read)
     @patch.object(Thread, 'start', Mock())
-    @patch.object(AbstractEtcdClientWithFailover, 'machines', PropertyMock(return_value=['http://remotehost:2379']))
+    @patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379']))
     def setUp(self):
         self._handlers = logging.getLogger().handlers[:]
         RestApiServer._BaseServer__is_shut_down = Mock()
@@ -88,7 +88,7 @@ class TestPatroni(unittest.TestCase):
     @patch('sys.argv', ['patroni.py', 'postgres0.yml'])
     @patch('time.sleep', Mock(side_effect=SleepException))
     @patch.object(etcd.Client, 'delete', Mock())
-    @patch.object(AbstractEtcdClientWithFailover, 'machines', PropertyMock(return_value=['http://remotehost:2379']))
+    @patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379']))
     @patch.object(Thread, 'join', Mock())
     def test_patroni_patroni_main(self):
         with patch('subprocess.call', Mock(return_value=1)):
@@ -196,7 +196,7 @@ class TestPatroni(unittest.TestCase):
         self.p.shutdown()
 
     def test_check_psycopg(self):
-        with patch.object(builtins, '__import__', Mock(side_effect=ImportError)):
+        with patch('builtins.__import__', Mock(side_effect=ImportError)):
             self.assertRaises(SystemExit, check_psycopg)
-        with patch.object(builtins, '__import__', mock_import):
+        with patch('builtins.__import__', mock_import):
             self.assertRaises(SystemExit, check_psycopg)

tests/test_postgresql.py

@@ -14,9 +14,9 @@ from patroni.dcs import RemoteMember
 from patroni.exceptions import PostgresConnectionException, PatroniException
 from patroni.postgresql import Postgresql, STATE_REJECT, STATE_NO_RESPONSE
 from patroni.postgresql.bootstrap import Bootstrap
+from patroni.postgresql.callback_executor import CallbackAction
 from patroni.postgresql.postmaster import PostmasterProcess
 from patroni.utils import RetryFailedError
-from six.moves import builtins
 from threading import Thread, current_thread
 
 from . import BaseTestPostgresql, MockCursor, MockPostmaster, psycopg_connect
@@ -114,6 +114,9 @@ class TestPostgresql(BaseTestPostgresql):
         self.assertTrue(self.p.start())
         mock_is_running.return_value = None
 
+        with patch.object(Postgresql, 'ensure_major_version_is_known', Mock(return_value=False)):
+            self.assertIsNone(self.p.start())
+
         mock_postmaster = MockPostmaster()
         with patch.object(PostmasterProcess, 'start', return_value=mock_postmaster):
             pg_conf = os.path.join(self.p.data_dir, 'postgresql.conf')
@@ -227,7 +230,7 @@ class TestPostgresql(BaseTestPostgresql):
         self.assertEqual(self.p.state, 'restart failed (restarting)')
 
     @patch('os.chmod', Mock())
-    @patch.object(builtins, 'open', MagicMock())
+    @patch('builtins.open', MagicMock())
     def test_write_pgpass(self):
         self.p.config.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo'})
         self.p.config.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo', 'password': 'bar'})
@@ -242,7 +245,7 @@ class TestPostgresql(BaseTestPostgresql):
     @patch('patroni.postgresql.config.mtime', mock_mtime)
     @patch('patroni.postgresql.config.ConfigHandler._get_pg_settings')
     def test_check_recovery_conf(self, mock_get_pg_settings):
-        self.p.call_nowait('on_start')
+        self.p.call_nowait(CallbackAction.ON_START)
         mock_get_pg_settings.return_value = {
             'primary_conninfo': ['primary_conninfo', 'foo=', None, 'string', 'postmaster', self.p.config._auto_conf],
             'recovery_min_apply_delay': ['recovery_min_apply_delay', '0', 'ms', 'integer', 'sighup', 'foo']
@@ -278,7 +281,7 @@ class TestPostgresql(BaseTestPostgresql):
     @patch.object(MockPostmaster, 'create_time', Mock(return_value=1234567), create=True)
     @patch('patroni.postgresql.config.ConfigHandler._get_pg_settings')
     def test__read_recovery_params(self, mock_get_pg_settings):
-        self.p.call_nowait('on_start')
+        self.p.call_nowait(CallbackAction.ON_START)
         mock_get_pg_settings.return_value = {'primary_conninfo': ['primary_conninfo', '', None, 'string',
                                                                   'postmaster', self.p.config._postgresql_conf]}
         self.p.config.write_recovery_conf({'standby_mode': 'on', 'primary_conninfo': {'password': 'foo'}})
@@ -308,11 +311,11 @@ class TestPostgresql(BaseTestPostgresql):
 
         mock_read_auto = mock_open(read_data=read_data)
         mock_read_auto.return_value.__iter__ = lambda o: iter(o.readline, '')
-        with patch.object(builtins, 'open', Mock(side_effect=[mock_open()(), mock_read_auto(), IOError])),\
+        with patch('builtins.open', Mock(side_effect=[mock_open()(), mock_read_auto(), IOError])),\
                 patch('os.chmod', Mock()):
             self.p.config.write_postgresql_conf()
 
-        with patch.object(builtins, 'open', Mock(side_effect=[mock_open()(), IOError])), patch('os.chmod', Mock()):
+        with patch('builtins.open', Mock(side_effect=[mock_open()(), IOError])), patch('os.chmod', Mock()):
             self.p.config.write_postgresql_conf()
         self.p.config.write_recovery_conf({'foo': 'bar'})
         self.p.config.write_postgresql_conf()
@@ -320,9 +323,11 @@ class TestPostgresql(BaseTestPostgresql):
     @patch.object(Postgresql, 'is_running', Mock(return_value=False))
     @patch.object(Postgresql, 'start', Mock())
     def test_follow(self):
-        self.p.call_nowait('on_start')
+        self.p.call_nowait(CallbackAction.ON_START)
         m = RemoteMember('1', {'restore_command': '2', 'primary_slot_name': 'foo', 'conn_kwargs': {'host': 'bar'}})
         self.p.follow(m)
+        with patch.object(Postgresql, 'ensure_major_version_is_known', Mock(return_value=False)):
+            self.assertIsNone(self.p.follow(m))
 
     @patch.object(MockCursor, 'execute', Mock(side_effect=psycopg.OperationalError))
     def test__query(self):
@@ -425,12 +430,9 @@ class TestPostgresql(BaseTestPostgresql):
     @patch('shlex.split', Mock(side_effect=OSError))
     def test_call_nowait(self):
         self.p.set_role('replica')
-        self.assertIsNone(self.p.call_nowait('on_start'))
+        self.assertIsNone(self.p.call_nowait(CallbackAction.ON_START))
         self.p.bootstrapping = True
-        self.assertIsNone(self.p.call_nowait('on_start'))
+        self.assertIsNone(self.p.call_nowait(CallbackAction.ON_START))
-
-    def test_non_existing_callback(self):
-        self.assertFalse(self.p.call_nowait('foobar'))
 
     @patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
     def test_is_leader_exception(self):
@@ -549,9 +551,9 @@ class TestPostgresql(BaseTestPostgresql):
 
     @patch.object(Postgresql, '_version_file_exists', Mock(return_value=True))
     def test_get_major_version(self):
-        with patch.object(builtins, 'open', mock_open(read_data='9.4')):
+        with patch('builtins.open', mock_open(read_data='9.4')):
             self.assertEqual(self.p.get_major_version(), 90400)
-        with patch.object(builtins, 'open', Mock(side_effect=Exception)):
+        with patch('builtins.open', Mock(side_effect=Exception)):
             self.assertEqual(self.p.get_major_version(), 0)
 
     def test_postmaster_start_time(self):
@@ -675,14 +677,20 @@ class TestPostgresql(BaseTestPostgresql):
 
     @patch.object(Postgresql, 'get_postgres_role_from_data_directory', Mock(return_value='replica'))
     @patch.object(Bootstrap, 'running_custom_bootstrap', PropertyMock(return_value=True))
-    @patch.object(Bootstrap, 'keep_existing_recovery_conf', PropertyMock(return_value=True))
+    @patch.object(Postgresql, 'controldata', Mock(return_value={'max_connections setting': '200',
-    def test__build_effective_configuration(self):
+                                                                'max_worker_processes setting': '20',
-        with patch.object(Postgresql, 'controldata',
+                                                                'max_locks_per_xact setting': '100',
-                          Mock(return_value={'max_connections setting': '200',
+                                                                'max_wal_senders setting': 10}))
-                                             'max_worker_processes setting': '20',
+    @patch('patroni.postgresql.config.logger.warning')
-                                             'max_locks_per_xact setting': '100',
+    def test_effective_configuration(self, mock_logger):
-                                             'max_wal_senders setting': 10})):
+        self.p.cancellable.cancel()
-            self.p.cancellable.cancel()
+        self.p.config.write_recovery_conf({'pause_at_recovery_target': 'false'})
+        self.assertFalse(self.p.start())
+        mock_logger.assert_called_once()
+        self.assertTrue('is missing from pg_controldata output' in mock_logger.call_args[0][0])
+
+        self.assertTrue(self.p.pending_restart)
+        with patch.object(Bootstrap, 'keep_existing_recovery_conf', PropertyMock(return_value=True)):
             self.assertFalse(self.p.start())
             self.assertTrue(self.p.pending_restart)
 

tests/test_postmaster.py

@@ -4,7 +4,6 @@ import unittest
 
 from mock import Mock, patch, mock_open
 from patroni.postgresql.postmaster import PostmasterProcess
-from six.moves import builtins
 
 
 class MockProcess(object):
@@ -169,7 +168,7 @@ class TestPostmasterProcess(unittest.TestCase):
 
     @patch('psutil.Process.__init__', Mock(side_effect=psutil.NoSuchProcess(123)))
     def test_read_postmaster_pidfile(self):
-        with patch.object(builtins, 'open', Mock(side_effect=IOError)):
+        with patch('builtins.open', Mock(side_effect=IOError)):
             self.assertIsNone(PostmasterProcess.from_pidfile(''))
-        with patch.object(builtins, 'open', mock_open(read_data='123\n')):
+        with patch('builtins.open', mock_open(read_data='123\n')):
             self.assertIsNone(PostmasterProcess.from_pidfile(''))

tests/test_rewind.py

@@ -3,7 +3,6 @@ from mock import Mock, PropertyMock, patch, mock_open
 from patroni.postgresql import Postgresql
 from patroni.postgresql.cancellable import CancellableSubprocess
 from patroni.postgresql.rewind import Rewind
-from six.moves import builtins
 
 from . import BaseTestPostgresql, MockCursor, psycopg_connect
 
@@ -193,7 +192,7 @@ class TestRewind(BaseTestPostgresql):
         m = mock_open(read_data='/usr/lib/postgres/9.6/bin/postgres "-D" "data/postgresql0" \
 "--listen_addresses=127.0.0.1" "--port=5432" "--hot_standby=on" "--wal_level=hot_standby" \
 "--wal_log_hints=on" "--max_wal_senders=5" "--max_replication_slots=5"\n')
-        with patch.object(builtins, 'open', m):
+        with patch('builtins.open', m):
             data = self.r.read_postmaster_opts()
             self.assertEqual(data['wal_level'], 'hot_standby')
             self.assertEqual(int(data['max_replication_slots']), 5)

tests/test_slots.py

@@ -7,7 +7,7 @@ from mock import Mock, PropertyMock, patch
 from threading import Thread
 
 from patroni import psycopg
-from patroni.dcs import Cluster, ClusterConfig, Member
+from patroni.dcs import Cluster, ClusterConfig, Member, SyncState
 from patroni.postgresql import Postgresql
 from patroni.postgresql.misc import fsync_dir
 from patroni.postgresql.slots import SlotsAdvanceThread, SlotsHandler
@@ -31,15 +31,15 @@ class TestSlotsHandler(BaseTestPostgresql):
         self.s = self.p.slots_handler
         self.p.start()
         config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}}}, 1)
-        self.cluster = Cluster(True, config, self.leader, 0,
+        self.cluster = Cluster(True, config, self.leader, 0, [self.me, self.other, self.leadermem],
-                               [self.me, self.other, self.leadermem], None, None, None, {'ls': 12345}, None)
+                               None, SyncState.empty(), None, {'ls': 12345}, None)
 
     def test_sync_replication_slots(self):
         config = ClusterConfig(1, {'slots': {'test_3': {'database': 'a', 'plugin': 'b'},
                                              'A': 0, 'ls': 0, 'b': {'type': 'logical', 'plugin': '1'}},
                                    'ignore_slots': [{'name': 'blabla'}]}, 1)
-        cluster = Cluster(True, config, self.leader, 0,
+        cluster = Cluster(True, config, self.leader, 0, [self.me, self.other, self.leadermem],
-                          [self.me, self.other, self.leadermem], None, None, None, {'test_3': 10}, None)
+                          None, SyncState.empty(), None, {'test_3': 10}, None)
         with mock.patch('patroni.postgresql.Postgresql._query', Mock(side_effect=psycopg.OperationalError)):
             self.s.sync_replication_slots(cluster, False)
         self.p.set_role('standby_leader')
@@ -70,8 +70,8 @@ class TestSlotsHandler(BaseTestPostgresql):
     def test_process_permanent_slots(self):
         config = ClusterConfig(1, {'slots': {'ls': {'database': 'a', 'plugin': 'b'}},
                                    'ignore_slots': [{'name': 'blabla'}]}, 1)
-        cluster = Cluster(True, config, self.leader, 0,
+        cluster = Cluster(True, config, self.leader, 0, [self.me, self.other, self.leadermem],
-                          [self.me, self.other, self.leadermem], None, None, None, None, None)
+                          None, SyncState.empty(), None, None, None)
 
         self.s.sync_replication_slots(cluster, False)
         with patch.object(Postgresql, '_query') as mock_query:

tests/test_utils.py

@@ -2,7 +2,7 @@ import unittest
 
 from mock import Mock, patch
 from patroni.exceptions import PatroniException
-from patroni.utils import Retry, RetryFailedError, enable_keepalive, find_executable, polling_loop, validate_directory
+from patroni.utils import Retry, RetryFailedError, enable_keepalive, polling_loop, validate_directory
 
 
 class TestUtils(unittest.TestCase):
@@ -41,15 +41,6 @@ class TestUtils(unittest.TestCase):
                 with patch('sys.platform', platform):
                     self.assertIsNone(enable_keepalive(Mock(), 10, 5))
 
-    @patch('sys.platform', 'win32')
-    def test_find_executable(self):
-        with patch('os.path.isfile', Mock(return_value=True)):
-            self.assertEqual(find_executable('vim'), 'vim.exe')
-        with patch('os.path.isfile', Mock(return_value=False)):
-            self.assertIsNone(find_executable('vim'))
-        with patch('os.path.isfile', Mock(side_effect=[False, True])):
-            self.assertEqual(find_executable('vim', '/'), '/vim.exe')
-
 
 @patch('time.sleep', Mock())
 class TestRetrySleeper(unittest.TestCase):

tests/test_validator.py

@@ -4,10 +4,10 @@ import socket
 import tempfile
 import unittest
 
+from io import StringIO
 from mock import Mock, patch, mock_open
 from patroni.dcs import dcs_modules
 from patroni.validator import schema
-from six import StringIO
 
 available_dcs = [m.split(".")[-1] for m in dcs_modules()]
 config = {
@@ -59,6 +59,7 @@ config = {
         "use_endpoints": False,
         "pod_ip": "127.0.0.1",
         "ports": [{"name": "string", "port": 1000}],
+        "retriable_http_codes": [401],
     },
     "postgresql": {
         "listen": "127.0.0.2,::1:543",
@@ -93,14 +94,18 @@ config = {
 
 directories = []
 files = []
+binaries = []
 
 
 def isfile_side_effect(arg):
-    if arg.endswith('.exe'):
-        arg = arg[:-4]
     return arg in files
 
 
+def which_side_effect(arg, path=None):
+    binary = arg if path is None else os.path.join(path, arg)
+    return arg if binary in binaries else None
+
+
 def isdir_side_effect(arg):
     return arg in directories
 
@@ -133,6 +138,7 @@ def parse_output(output):
 @patch('os.path.exists', Mock(side_effect=exists_side_effect))
 @patch('os.path.isdir', Mock(side_effect=isdir_side_effect))
 @patch('os.path.isfile', Mock(side_effect=isfile_side_effect))
+@patch('shutil.which', Mock(side_effect=which_side_effect))
 @patch('sys.stderr', new_callable=StringIO)
 @patch('sys.stdout', new_callable=StringIO)
 class TestValidator(unittest.TestCase):
@@ -140,6 +146,7 @@ class TestValidator(unittest.TestCase):
     def setUp(self):
         del files[:]
         del directories[:]
+        del binaries[:]
 
     def test_empty_config(self, mock_out, mock_err):
         errors = schema({})
@@ -190,12 +197,12 @@ class TestValidator(unittest.TestCase):
         directories.append(os.path.join(config["postgresql"]["data_dir"], "pg_wal"))
         files.append(os.path.join(config["postgresql"]["data_dir"], "global", "pg_control"))
         files.append(os.path.join(config["postgresql"]["data_dir"], "PG_VERSION"))
-        files.append(os.path.join(config["postgresql"]["bin_dir"], "pg_ctl"))
+        binaries.append(os.path.join(config["postgresql"]["bin_dir"], "pg_ctl"))
-        files.append(os.path.join(config["postgresql"]["bin_dir"], "initdb"))
+        binaries.append(os.path.join(config["postgresql"]["bin_dir"], "initdb"))
-        files.append(os.path.join(config["postgresql"]["bin_dir"], "pg_controldata"))
+        binaries.append(os.path.join(config["postgresql"]["bin_dir"], "pg_controldata"))
-        files.append(os.path.join(config["postgresql"]["bin_dir"], "pg_basebackup"))
+        binaries.append(os.path.join(config["postgresql"]["bin_dir"], "pg_basebackup"))
-        files.append(os.path.join(config["postgresql"]["bin_dir"], "postgres"))
+        binaries.append(os.path.join(config["postgresql"]["bin_dir"], "postgres"))
-        files.append(os.path.join(config["postgresql"]["bin_dir"], "pg_isready"))
+        binaries.append(os.path.join(config["postgresql"]["bin_dir"], "pg_isready"))
         with patch('patroni.validator.open', mock_open(read_data='12')):
             errors = schema(config)
         output = "\n".join(errors)

tests/test_wale_restore.py

@@ -6,7 +6,6 @@ import patroni.psycopg as psycopg
 from mock import Mock, PropertyMock, patch, mock_open
 from patroni.scripts import wale_restore
 from patroni.scripts.wale_restore import WALERestore, main as _main, get_major_version
-from six.moves import builtins
 from threading import current_thread
 
 from . import MockConnect, psycopg_connect
@@ -128,9 +127,9 @@ class TestWALERestore(unittest.TestCase):
 
     @patch('os.path.isfile', Mock(return_value=True))
     def test_get_major_version(self):
-        with patch.object(builtins, 'open', mock_open(read_data='9.4')):
+        with patch('builtins.open', mock_open(read_data='9.4')):
             self.assertEqual(get_major_version("data"), 9.4)
-        with patch.object(builtins, 'open', side_effect=OSError):
+        with patch('builtins.open', side_effect=OSError):
             self.assertEqual(get_major_version("data"), 0.0)
 
     @patch('os.path.islink', Mock(return_value=True))

tests/test_zookeeper.py

@@ -1,5 +1,4 @@
 import select
-import six
 import unittest
 
 from kazoo.client import KazooClient, KazooState
@@ -30,7 +29,7 @@ class MockKazooClient(Mock):
         return func(*args, **kwargs)
 
     def get(self, path, watch=None):
-        if not isinstance(path, six.string_types):
+        if not isinstance(path, str):
             raise TypeError("Invalid type for 'path' (string expected)")
         if path == '/broken/status':
             return (b'{', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0))
@@ -57,7 +56,7 @@ class MockKazooClient(Mock):
 
     @staticmethod
     def get_children(path, watch=None, include_data=False):
-        if not isinstance(path, six.string_types):
+        if not isinstance(path, str):
             raise TypeError("Invalid type for 'path' (string expected)")
         if path.startswith('/no_node'):
             raise NoNodeError
@@ -66,9 +65,9 @@ class MockKazooClient(Mock):
         return ['foo', 'bar', 'buzz']
 
     def create(self, path, value=b"", acl=None, ephemeral=False, sequence=False, makepath=False):
-        if not isinstance(path, six.string_types):
+        if not isinstance(path, str):
             raise TypeError("Invalid type for 'path' (string expected)")
-        if not isinstance(value, (six.binary_type,)):
+        if not isinstance(value, bytes):
             raise TypeError("Invalid type for 'value' (must be a byte string)")
         if b'Exception' in value:
             raise Exception
@@ -82,9 +81,9 @@ class MockKazooClient(Mock):
 
     @staticmethod
     def set(path, value, version=-1):
-        if not isinstance(path, six.string_types):
+        if not isinstance(path, str):
             raise TypeError("Invalid type for 'path' (string expected)")
-        if not isinstance(value, (six.binary_type,)):
+        if not isinstance(value, bytes):
             raise TypeError("Invalid type for 'value' (must be a byte string)")
         if path == '/service/bla/optime/leader':
             raise Exception
@@ -101,7 +100,7 @@ class MockKazooClient(Mock):
         return self.set(path, value, version) or Mock()
 
     def delete(self, path, version=-1, recursive=False):
-        if not isinstance(path, six.string_types):
+        if not isinstance(path, str):
             raise TypeError("Invalid type for 'path' (string expected)")
         self.exists = False
         if path == '/service/test/leader':